2024-03-29T04:13:53.582 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-29T04:13:53.583 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108 --verbose --name teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/orig.config.yaml 2024-03-29T04:13:53.646 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-29T04:13:53.733 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7629108' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4408 sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 19/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba targets: smithi029.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJI7iNJ3YgHdkr+lBiW/B2AtbNJP5K6CIfLSCFpmyc0a6Ts/vecmiRiNTrYQ1Nnknsr4Ybi3eegCYRcXWF9qJO4= smithi099.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMTGQ7qxV9Ue7P8GrTfDR8/kOj2hexkGczvRuhjHopL6xl53lYWmHrQvo2QWmC5hZK0bEwaEvVyO0/HgVaYXNIg= tasks: - pexec: all: - sudo dnf install runc -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-29_02:08:11 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-29T04:13:53.733 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa; will attempt to use it 2024-03-29T04:13:53.734 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks 2024-03-29T04:13:53.734 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-29T04:13:53.736 INFO:teuthology.task.internal:Checking packages... 2024-03-29T04:13:53.758 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '8da23624bc17fe0b1f797227e72a0db0ce7821ba' 2024-03-29T04:13:53.759 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:13:53.759 INFO:teuthology.packaging:ref: None 2024-03-29T04:13:53.759 INFO:teuthology.packaging:tag: None 2024-03-29T04:13:53.759 INFO:teuthology.packaging:branch: squid 2024-03-29T04:13:53.759 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:13:53.759 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-03-29T04:13:53.920 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1959.g8da23624 2024-03-29T04:13:53.922 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-29T04:13:53.929 INFO:teuthology.task.internal:no buildpackages task found 2024-03-29T04:13:53.929 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-29T04:13:53.942 INFO:teuthology.task.internal:Saving configuration 2024-03-29T04:13:53.964 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-29T04:13:53.971 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T04:13:53.996 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi029.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:08:34.354862', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJI7iNJ3YgHdkr+lBiW/B2AtbNJP5K6CIfLSCFpmyc0a6Ts/vecmiRiNTrYQ1Nnknsr4Ybi3eegCYRcXWF9qJO4='} 2024-03-29T04:13:54.018 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:08:34.356336', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMTGQ7qxV9Ue7P8GrTfDR8/kOj2hexkGczvRuhjHopL6xl53lYWmHrQvo2QWmC5hZK0bEwaEvVyO0/HgVaYXNIg='} 2024-03-29T04:13:54.018 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-29T04:13:54.025 INFO:teuthology.task.internal:roles: ubuntu@smithi029.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-03-29T04:13:54.025 INFO:teuthology.task.internal:roles: ubuntu@smithi099.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-03-29T04:13:54.025 INFO:teuthology.run_tasks:Running task console_log... 2024-03-29T04:13:54.093 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0fadb360d0>, signals=[15]) 2024-03-29T04:13:54.093 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-29T04:13:54.099 INFO:teuthology.task.internal:Opening connections... 2024-03-29T04:13:54.100 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi029.front.sepia.ceph.com 2024-03-29T04:13:54.101 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:13:54.211 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi099.front.sepia.ceph.com 2024-03-29T04:13:54.212 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:13:54.302 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-29T04:13:54.309 DEBUG:teuthology.orchestra.run.smithi029:> uname -m 2024-03-29T04:13:54.327 INFO:teuthology.orchestra.run.smithi029.stdout:x86_64 2024-03-29T04:13:54.327 DEBUG:teuthology.orchestra.run.smithi029:> cat /etc/os-release 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:NAME="CentOS Stream" 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:VERSION="9" 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:ID="centos" 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:ID_LIKE="rhel fedora" 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:VERSION_ID="9" 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:PLATFORM_ID="platform:el9" 2024-03-29T04:13:54.386 INFO:teuthology.orchestra.run.smithi029.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:ANSI_COLOR="0;31" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:LOGO="fedora-logo-icon" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:HOME_URL="https://centos.org/" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-29T04:13:54.387 INFO:teuthology.orchestra.run.smithi029.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-29T04:13:54.388 INFO:teuthology.lock.ops:Updating smithi029.front.sepia.ceph.com on lock server 2024-03-29T04:13:54.416 DEBUG:teuthology.orchestra.run.smithi099:> uname -m 2024-03-29T04:13:54.432 INFO:teuthology.orchestra.run.smithi099.stdout:x86_64 2024-03-29T04:13:54.432 DEBUG:teuthology.orchestra.run.smithi099:> cat /etc/os-release 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:NAME="CentOS Stream" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION="9" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:ID="centos" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:ID_LIKE="rhel fedora" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION_ID="9" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:PLATFORM_ID="platform:el9" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:ANSI_COLOR="0;31" 2024-03-29T04:13:54.488 INFO:teuthology.orchestra.run.smithi099.stdout:LOGO="fedora-logo-icon" 2024-03-29T04:13:54.489 INFO:teuthology.orchestra.run.smithi099.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-29T04:13:54.489 INFO:teuthology.orchestra.run.smithi099.stdout:HOME_URL="https://centos.org/" 2024-03-29T04:13:54.489 INFO:teuthology.orchestra.run.smithi099.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-29T04:13:54.489 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-29T04:13:54.489 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-29T04:13:54.489 INFO:teuthology.lock.ops:Updating smithi099.front.sepia.ceph.com on lock server 2024-03-29T04:13:54.507 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-29T04:13:54.516 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-29T04:13:54.610 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-29T04:13:54.610 DEBUG:teuthology.orchestra.run.smithi029:> test '!' -e /home/ubuntu/cephtest 2024-03-29T04:13:54.613 DEBUG:teuthology.orchestra.run.smithi099:> test '!' -e /home/ubuntu/cephtest 2024-03-29T04:13:54.628 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-29T04:13:54.633 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-29T04:13:54.634 DEBUG:teuthology.orchestra.run.smithi029:> test -z $(ls -A /var/lib/ceph) 2024-03-29T04:13:54.669 DEBUG:teuthology.orchestra.run.smithi099:> test -z $(ls -A /var/lib/ceph) 2024-03-29T04:13:54.742 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-29T04:13:54.794 INFO:teuthology.run_tasks:Running task kernel... 2024-03-29T04:13:54.808 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-29T04:13:54.808 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-29T04:13:54.809 DEBUG:teuthology.orchestra.run.smithi029:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-29T04:13:54.809 DEBUG:teuthology.orchestra.run.smithi099:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-29T04:13:54.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:13:54.826 DEBUG:teuthology.orchestra.run.smithi029:> uname -r 2024-03-29T04:13:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:13:54.827 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2024-03-29T04:13:54.883 INFO:teuthology.orchestra.run.smithi099.stdout:5.14.0-361.el9.x86_64 2024-03-29T04:13:54.884 INFO:teuthology.task.kernel:Running kernel on smithi099: 5.14.0-361.el9.x86_64 2024-03-29T04:13:54.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum install -y kernel 2024-03-29T04:13:54.885 INFO:teuthology.orchestra.run.smithi029.stdout:5.14.0-361.el9.x86_64 2024-03-29T04:13:54.885 INFO:teuthology.task.kernel:Running kernel on smithi029: 5.14.0-361.el9.x86_64 2024-03-29T04:13:54.885 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum install -y kernel 2024-03-29T04:13:57.043 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - AppStream 14 MB/s | 6.7 MB 00:00 2024-03-29T04:13:58.040 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-03-29T04:13:58.652 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - BaseOS 4.0 MB/s | 2.0 MB 00:00 2024-03-29T04:13:59.699 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - BaseOS 4.1 MB/s | 2.0 MB 00:00 2024-03-29T04:14:00.509 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.0 MB 00:00 2024-03-29T04:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - BaseOS 3.4 MB/s | 8.0 MB 00:02 2024-03-29T04:14:04.894 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - AppStream 5.8 MB/s | 19 MB 00:03 2024-03-29T04:14:09.279 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - CRB 8.3 MB/s | 6.1 MB 00:00 2024-03-29T04:14:10.840 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 16 kB 00:00 2024-03-29T04:14:12.136 INFO:teuthology.orchestra.run.smithi029.stdout:Extra Packages for Enterprise Linux 18 MB/s | 21 MB 00:01 2024-03-29T04:14:17.362 INFO:teuthology.orchestra.run.smithi029.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-03-29T04:14:19.745 INFO:teuthology.orchestra.run.smithi029.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-29T04:14:19.745 INFO:teuthology.orchestra.run.smithi029.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-29T04:14:19.850 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:14:19.865 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:14:19.865 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout:Install 4 Packages 2024-03-29T04:14:19.866 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:14:19.867 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 76 M 2024-03-29T04:14:19.867 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 123 M 2024-03-29T04:14:19.867 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:14:20.023 INFO:teuthology.orchestra.run.smithi029.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-03-29T04:14:20.665 INFO:teuthology.orchestra.run.smithi029.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 19 MB/s | 15 MB 00:00 2024-03-29T04:14:21.525 INFO:teuthology.orchestra.run.smithi029.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 20 MB/s | 34 MB 00:01 2024-03-29T04:14:22.159 INFO:teuthology.orchestra.run.smithi029.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 13 MB/s | 28 MB 00:02 2024-03-29T04:14:22.163 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:14:22.163 INFO:teuthology.orchestra.run.smithi029.stdout:Total 33 MB/s | 76 MB 00:02 2024-03-29T04:14:22.163 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:14:22.635 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:14:22.635 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:14:23.483 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:14:23.483 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:14:23.722 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - AppStream 912 kB/s | 19 MB 00:21 2024-03-29T04:14:25.910 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:14:26.603 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:14:26.704 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:14:28.106 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - CRB 7.9 MB/s | 6.1 MB 00:00 2024-03-29T04:14:29.551 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:14:29.699 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 16 kB 00:00 2024-03-29T04:14:29.750 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:14:31.566 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-03-29T04:14:34.828 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:14:34.871 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:14:36.449 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-03-29T04:14:38.714 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-29T04:14:38.714 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-29T04:14:38.806 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:14:38.814 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:14:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:14:38.814 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:14:38.814 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:14:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout:Install 4 Packages 2024-03-29T04:14:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:14:38.816 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 76 M 2024-03-29T04:14:38.816 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 123 M 2024-03-29T04:14:38.816 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:14:38.959 INFO:teuthology.orchestra.run.smithi099.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-03-29T04:14:39.468 INFO:teuthology.orchestra.run.smithi099.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 23 MB/s | 15 MB 00:00 2024-03-29T04:14:40.084 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:14:40.101 INFO:teuthology.orchestra.run.smithi099.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 26 MB/s | 34 MB 00:01 2024-03-29T04:14:40.359 INFO:teuthology.orchestra.run.smithi099.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 20 MB/s | 28 MB 00:01 2024-03-29T04:14:40.362 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:14:40.363 INFO:teuthology.orchestra.run.smithi099.stdout:Total 49 MB/s | 76 MB 00:01 2024-03-29T04:14:40.363 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:14:40.737 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:14:40.737 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:14:41.583 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:14:41.583 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:14:43.878 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:14:44.481 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:14:44.569 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:14:47.290 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:14:47.432 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:14:52.142 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:14:52.189 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:14:56.992 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:13.496 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:13.496 INFO:teuthology.orchestra.run.smithi029.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-29T04:15:13.496 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:15:13.540 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:13.964 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:13.964 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:15:13.964 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:15:13.964 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-29T04:15:14.403 INFO:teuthology.orchestra.run.smithi029.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-29T04:15:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:15:14.404 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:15:14.832 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:15:14.963 INFO:teuthology.orchestra.run.smithi029.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:15:14.963 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:15:14.964 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-29T04:15:14.964 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-29T04:15:14.964 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-29T04:15:14.964 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-03-29T04:15:14.964 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi029.front.sepia.ceph.com, path=None, version=distro) 2024-03-29T04:15:14.964 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:15:15.047 INFO:teuthology.orchestra.run.smithi029.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:15:15.048 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:15:15.048 DEBUG:teuthology.orchestra.run.smithi029:> sudo rpm -qi grub2-tools 2024-03-29T04:15:15.094 INFO:teuthology.orchestra.run.smithi029.stdout:Name : grub2-tools 2024-03-29T04:15:15.094 INFO:teuthology.orchestra.run.smithi029.stdout:Epoch : 1 2024-03-29T04:15:15.094 INFO:teuthology.orchestra.run.smithi029.stdout:Version : 2.06 2024-03-29T04:15:15.094 INFO:teuthology.orchestra.run.smithi029.stdout:Release : 68.el9 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Architecture: x86_64 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Group : Unspecified 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Size : 8264254 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:License : GPLv3+ 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Packager : builder@centos.org 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Vendor : CentOS 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Summary : Support tools for GRUB. 2024-03-29T04:15:15.095 INFO:teuthology.orchestra.run.smithi029.stdout:Description : 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout:hardware devices. 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:15:15.096 INFO:teuthology.orchestra.run.smithi029.stdout:This subpackage provides tools for support of all platforms. 2024-03-29T04:15:15.097 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-29T04:15:15.097 INFO:teuthology.task.kernel:Updating grub on smithi029 to boot 5.14.0-434.el9.x86_64 2024-03-29T04:15:15.097 DEBUG:teuthology.orchestra.run.smithi029:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-29T04:15:15.889 INFO:teuthology.orchestra.run.smithi029.stderr:Generating grub configuration file ... 2024-03-29T04:15:17.483 INFO:teuthology.orchestra.run.smithi029.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-29T04:15:17.509 INFO:teuthology.orchestra.run.smithi029.stderr:done 2024-03-29T04:15:17.512 DEBUG:teuthology.orchestra.run.smithi029:> mktemp 2024-03-29T04:15:17.527 INFO:teuthology.orchestra.run.smithi029.stdout:/tmp/tmp.2il66PAlFC 2024-03-29T04:15:17.527 DEBUG:teuthology.orchestra.run.smithi029:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.2il66PAlFC 2024-03-29T04:15:17.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 0666 /tmp/tmp.2il66PAlFC 2024-03-29T04:15:17.748 DEBUG:teuthology.orchestra.remote:smithi029:/tmp/tmp.2il66PAlFC is 6KB 2024-03-29T04:15:17.797 DEBUG:teuthology.orchestra.run.smithi029:> rm -fr /tmp/tmp.2il66PAlFC 2024-03-29T04:15:17.812 DEBUG:teuthology.orchestra.run.smithi029:> sudo /bin/ls /boot/loader/entries || true 2024-03-29T04:15:17.879 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-29T04:15:17.879 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-29T04:15:17.880 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-29T04:15:17.880 INFO:teuthology.orchestra.run.smithi029.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-29T04:15:17.881 DEBUG:teuthology.orchestra.run.smithi029:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-29T04:15:18.045 DEBUG:teuthology.orchestra.run.smithi029:> sudo shutdown -r now 2024-03-29T04:15:26.466 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:26.466 INFO:teuthology.orchestra.run.smithi099.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-29T04:15:26.466 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:15:26.508 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:26.835 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:26.835 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:15:26.835 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:15:26.835 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:15:27.240 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:15:27.563 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:15:27.672 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:15:27.672 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:15:27.672 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-29T04:15:27.673 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-29T04:15:27.673 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-29T04:15:27.673 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-03-29T04:15:27.673 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi099.front.sepia.ceph.com, path=None, version=distro) 2024-03-29T04:15:27.673 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:15:27.753 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:15:27.753 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:15:27.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo rpm -qi grub2-tools 2024-03-29T04:15:27.796 INFO:teuthology.orchestra.run.smithi099.stdout:Name : grub2-tools 2024-03-29T04:15:27.796 INFO:teuthology.orchestra.run.smithi099.stdout:Epoch : 1 2024-03-29T04:15:27.796 INFO:teuthology.orchestra.run.smithi099.stdout:Version : 2.06 2024-03-29T04:15:27.796 INFO:teuthology.orchestra.run.smithi099.stdout:Release : 68.el9 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Architecture: x86_64 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Group : Unspecified 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Size : 8264254 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:License : GPLv3+ 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Packager : builder@centos.org 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Vendor : CentOS 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-29T04:15:27.797 INFO:teuthology.orchestra.run.smithi099.stdout:Summary : Support tools for GRUB. 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:Description : 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:hardware devices. 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:15:27.798 INFO:teuthology.orchestra.run.smithi099.stdout:This subpackage provides tools for support of all platforms. 2024-03-29T04:15:27.799 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-29T04:15:27.799 INFO:teuthology.task.kernel:Updating grub on smithi099 to boot 5.14.0-434.el9.x86_64 2024-03-29T04:15:27.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-29T04:15:28.416 INFO:teuthology.orchestra.run.smithi099.stderr:Generating grub configuration file ... 2024-03-29T04:15:29.906 INFO:teuthology.orchestra.run.smithi099.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-29T04:15:29.945 INFO:teuthology.orchestra.run.smithi099.stderr:done 2024-03-29T04:15:29.954 DEBUG:teuthology.orchestra.run.smithi099:> mktemp 2024-03-29T04:15:29.971 INFO:teuthology.orchestra.run.smithi099.stdout:/tmp/tmp.e3vdeYq7pA 2024-03-29T04:15:29.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.e3vdeYq7pA 2024-03-29T04:15:30.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0666 /tmp/tmp.e3vdeYq7pA 2024-03-29T04:15:30.177 DEBUG:teuthology.orchestra.remote:smithi099:/tmp/tmp.e3vdeYq7pA is 6KB 2024-03-29T04:15:30.226 DEBUG:teuthology.orchestra.run.smithi099:> rm -fr /tmp/tmp.e3vdeYq7pA 2024-03-29T04:15:30.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo /bin/ls /boot/loader/entries || true 2024-03-29T04:15:30.305 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-29T04:15:30.305 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-29T04:15:30.305 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-29T04:15:30.305 INFO:teuthology.orchestra.run.smithi099.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-29T04:15:30.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-29T04:15:30.446 DEBUG:teuthology.orchestra.run.smithi099:> sudo shutdown -r now 2024-03-29T04:15:48.062 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-03-29T04:15:48.063 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-03-29T04:15:48.064 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:16:00.461 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-03-29T04:16:00.461 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-29T04:16:00.462 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:16:06.430 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.29 2024-03-29T04:16:15.440 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-03-29T04:16:15.441 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:16:35.103 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2024-03-29T04:16:44.110 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-29T04:16:44.111 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:17:02.499 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.99 2024-03-29T04:17:14.510 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-29T04:17:14.511 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:17:15.443 DEBUG:teuthology.orchestra.remote:timed out 2024-03-29T04:17:15.565 DEBUG:teuthology.orchestra.run.smithi099:> true 2024-03-29T04:17:16.084 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-29T04:17:16.084 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-434.el9.x86_64"... 2024-03-29T04:17:16.084 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2024-03-29T04:17:16.099 INFO:teuthology.orchestra.run.smithi099.stdout:5.14.0-434.el9.x86_64 2024-03-29T04:17:16.099 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-29T04:17:16.099 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-29T04:17:16.099 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-29T04:17:17.100 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-03-29T04:17:17.100 DEBUG:teuthology.orchestra.run.smithi099:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-29T04:17:17.191 INFO:teuthology.orchestra.run.smithi099.stdout:ttyS1 2024-03-29T04:17:17.227 DEBUG:teuthology.parallel:result is None 2024-03-29T04:17:27.444 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-03-29T04:17:27.445 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:17:27.880 DEBUG:teuthology.orchestra.run.smithi029:> true 2024-03-29T04:17:28.300 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-03-29T04:17:28.301 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-434.el9.x86_64"... 2024-03-29T04:17:28.301 DEBUG:teuthology.orchestra.run.smithi029:> uname -r 2024-03-29T04:17:28.317 INFO:teuthology.orchestra.run.smithi029.stdout:5.14.0-434.el9.x86_64 2024-03-29T04:17:28.317 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-29T04:17:28.317 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-29T04:17:28.317 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-29T04:17:29.318 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-03-29T04:17:29.318 DEBUG:teuthology.orchestra.run.smithi029:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-29T04:17:29.427 INFO:teuthology.orchestra.run.smithi029.stdout:ttyS1 2024-03-29T04:17:29.467 DEBUG:teuthology.parallel:result is None 2024-03-29T04:17:29.467 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-29T04:17:29.475 INFO:teuthology.task.internal:Creating test directory... 2024-03-29T04:17:29.475 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-29T04:17:29.478 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-29T04:17:29.497 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-29T04:17:29.503 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-29T04:17:29.510 INFO:teuthology.task.internal:Creating archive directory... 2024-03-29T04:17:29.510 DEBUG:teuthology.orchestra.run.smithi029:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-29T04:17:29.535 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-29T04:17:29.591 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-29T04:17:29.598 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-29T04:17:29.598 DEBUG:teuthology.orchestra.run.smithi029:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-29T04:17:29.605 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-29T04:17:29.638 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:17:29.639 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:17:29.652 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:17:29.653 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:17:29.655 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-29T04:17:29.662 INFO:teuthology.task.internal:Configuring sudo... 2024-03-29T04:17:29.662 DEBUG:teuthology.orchestra.run.smithi029:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-29T04:17:29.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-29T04:17:29.726 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-29T04:17:29.735 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-29T04:17:29.735 DEBUG:teuthology.orchestra.run.smithi029:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-29T04:17:29.767 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-29T04:17:29.783 DEBUG:teuthology.orchestra.run.smithi029:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:17:29.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:17:29.917 DEBUG:teuthology.orchestra.run.smithi029:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:17:30.000 DEBUG:teuthology.orchestra.run.smithi029:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:17:30.043 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:17:30.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-29T04:17:30.110 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:17:30.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:17:30.208 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:17:30.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:17:30.337 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:17:30.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-29T04:17:30.406 DEBUG:teuthology.orchestra.run.smithi029:> sudo service rsyslog restart 2024-03-29T04:17:30.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo service rsyslog restart 2024-03-29T04:17:30.461 INFO:teuthology.orchestra.run.smithi029.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:17:30.505 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:17:31.009 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-29T04:17:31.017 INFO:teuthology.task.internal:Starting timer... 2024-03-29T04:17:31.017 INFO:teuthology.run_tasks:Running task pcp... 2024-03-29T04:17:31.027 INFO:teuthology.run_tasks:Running task selinux... 2024-03-29T04:17:31.037 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-03-29T04:17:31.037 DEBUG:teuthology.orchestra.run.smithi029:> sudo service auditd rotate 2024-03-29T04:17:31.113 INFO:teuthology.orchestra.run.smithi029.stdout:Rotating logs: 2024-03-29T04:17:31.116 DEBUG:teuthology.orchestra.run.smithi099:> sudo service auditd rotate 2024-03-29T04:17:31.174 INFO:teuthology.orchestra.run.smithi099.stdout:Rotating logs: 2024-03-29T04:17:31.176 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-29T04:17:31.176 DEBUG:teuthology.orchestra.run.smithi029:> /usr/sbin/getenforce 2024-03-29T04:17:31.199 INFO:teuthology.orchestra.run.smithi029.stdout:Permissive 2024-03-29T04:17:31.199 DEBUG:teuthology.orchestra.run.smithi099:> /usr/sbin/getenforce 2024-03-29T04:17:31.226 INFO:teuthology.orchestra.run.smithi099.stdout:Permissive 2024-03-29T04:17:31.226 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi029.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi099.front.sepia.ceph.com': 'permissive'} 2024-03-29T04:17:31.227 DEBUG:teuthology.orchestra.run.smithi029:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:17:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:17:31.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:17:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:17:31.303 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-29T04:17:31.303 DEBUG:teuthology.orchestra.run.smithi029:> sudo /usr/sbin/setenforce permissive 2024-03-29T04:17:31.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo /usr/sbin/setenforce permissive 2024-03-29T04:17:31.394 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-29T04:17:31.404 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-29T04:17:31.424 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-29T04:17:31.425 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi029.front.sepia.ceph.com,smithi099.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-29T04:21:55.297 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi029.front.sepia.ceph.com'), Remote(name='ubuntu@smithi099.front.sepia.ceph.com')] 2024-03-29T04:21:55.299 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-03-29T04:21:55.301 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi029.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:21:55.406 DEBUG:teuthology.orchestra.run.smithi029:> true 2024-03-29T04:21:55.489 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi029.front.sepia.ceph.com' 2024-03-29T04:21:55.490 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-29T04:21:55.490 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:21:55.581 DEBUG:teuthology.orchestra.run.smithi099:> true 2024-03-29T04:21:55.657 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2024-03-29T04:21:55.658 INFO:teuthology.run_tasks:Running task clock... 2024-03-29T04:21:55.671 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-29T04:21:55.672 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-29T04:21:55.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:21:55.674 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-29T04:21:55.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:21:55.705 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-29T04:21:55.723 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-29T04:21:55.730 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-29T04:21:55.747 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-29T04:21:55.753 INFO:teuthology.orchestra.run.smithi029.stderr:sudo: ntpd: command not found 2024-03-29T04:21:55.767 INFO:teuthology.orchestra.run.smithi029.stdout:506 Cannot talk to daemon 2024-03-29T04:21:55.777 INFO:teuthology.orchestra.run.smithi099.stderr:sudo: ntpd: command not found 2024-03-29T04:21:55.784 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-29T04:21:55.791 INFO:teuthology.orchestra.run.smithi099.stdout:506 Cannot talk to daemon 2024-03-29T04:21:55.801 INFO:teuthology.orchestra.run.smithi029.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-29T04:21:55.807 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-29T04:21:55.824 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-29T04:21:55.857 INFO:teuthology.orchestra.run.smithi029.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:21:55.860 INFO:teuthology.orchestra.run.smithi029.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:21:55.860 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================== 2024-03-29T04:21:55.861 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.861 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.861 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.861 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.883 INFO:teuthology.orchestra.run.smithi099.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:21:55.887 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:21:55.887 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2024-03-29T04:21:55.887 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.887 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.887 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.887 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:21:55.887 INFO:teuthology.run_tasks:Running task pexec... 2024-03-29T04:21:55.899 INFO:teuthology.task.pexec:Executing custom commands... 2024-03-29T04:21:55.900 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi029.front.sepia.ceph.com 2024-03-29T04:21:55.900 DEBUG:teuthology.orchestra.run.smithi029:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-03-29T04:21:55.900 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi099.front.sepia.ceph.com 2024-03-29T04:21:55.900 DEBUG:teuthology.orchestra.run.smithi099:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-03-29T04:21:56.498 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:48 ago on Fri 29 Mar 2024 04:21:08 AM UTC. 2024-03-29T04:21:56.509 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:01 ago on Fri 29 Mar 2024 04:20:55 AM UTC. 2024-03-29T04:21:56.652 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:21:56.656 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:21:56.667 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:21:56.667 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2024-03-29T04:21:56.667 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:21:56.667 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:21:56.667 INFO:teuthology.orchestra.run.smithi029.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 3.1 M 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 9.8 M 2024-03-29T04:21:56.668 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:21:56.673 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:21:56.674 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 3.1 M 2024-03-29T04:21:56.674 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 9.8 M 2024-03-29T04:21:56.674 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:21:57.518 INFO:teuthology.orchestra.run.smithi099.stdout:runc-1.1.12-2.el9.x86_64.rpm 4.3 MB/s | 3.1 MB 00:00 2024-03-29T04:21:57.518 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:21:57.518 INFO:teuthology.orchestra.run.smithi099.stdout:Total 3.7 MB/s | 3.1 MB 00:00 2024-03-29T04:21:57.554 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:21:57.560 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:21:57.560 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:21:57.638 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:21:57.639 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:21:57.654 INFO:teuthology.orchestra.run.smithi029.stdout:runc-1.1.12-2.el9.x86_64.rpm 3.4 MB/s | 3.1 MB 00:00 2024-03-29T04:21:57.655 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:21:57.655 INFO:teuthology.orchestra.run.smithi029.stdout:Total 3.2 MB/s | 3.1 MB 00:00 2024-03-29T04:21:57.694 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:21:57.701 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:21:57.701 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:21:57.786 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:21:57.786 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:21:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:21:58.025 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:21:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:21:58.130 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:21:58.471 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:21:58.801 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:21:58.801 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:21:58.801 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:21:58.801 INFO:teuthology.orchestra.run.smithi099.stdout: runc-4:1.1.12-2.el9.x86_64 2024-03-29T04:21:58.801 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:21:58.801 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:21:58.872 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:21:59.080 DEBUG:teuthology.parallel:result is None 2024-03-29T04:21:59.217 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:21:59.217 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:21:59.217 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:21:59.217 INFO:teuthology.orchestra.run.smithi029.stdout: runc-4:1.1.12-2.el9.x86_64 2024-03-29T04:21:59.217 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:21:59.217 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:21:59.536 DEBUG:teuthology.parallel:result is None 2024-03-29T04:21:59.537 INFO:teuthology.run_tasks:Running task install... 2024-03-29T04:21:59.547 DEBUG:teuthology.task.install:project ceph 2024-03-29T04:21:59.547 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'}} 2024-03-29T04:21:59.547 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'} 2024-03-29T04:21:59.547 INFO:teuthology.task.install:Using flavor: default 2024-03-29T04:21:59.553 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-29T04:21:59.553 INFO:teuthology.task.install:extra packages: [] 2024-03-29T04:21:59.553 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba', 'tag': None, 'wait_for_package': False} 2024-03-29T04:21:59.553 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:21:59.554 INFO:teuthology.packaging:ref: None 2024-03-29T04:21:59.554 INFO:teuthology.packaging:tag: None 2024-03-29T04:21:59.554 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:21:59.554 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:21:59.554 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:21:59.557 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba', 'tag': None, 'wait_for_package': False} 2024-03-29T04:21:59.557 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:21:59.557 INFO:teuthology.packaging:ref: None 2024-03-29T04:21:59.557 INFO:teuthology.packaging:tag: None 2024-03-29T04:21:59.557 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:21:59.557 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:21:59.557 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:21:59.867 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/ 2024-03-29T04:21:59.868 INFO:teuthology.task.install.rpm:Package version is 17.2.7-767.g9b4b8ffd 2024-03-29T04:21:59.881 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/ 2024-03-29T04:21:59.882 INFO:teuthology.task.install.rpm:Package version is 17.2.7-767.g9b4b8ffd 2024-03-29T04:21:59.990 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-29T04:21:59.991 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:21:59.991 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-29T04:22:00.020 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-03-29T04:22:00.020 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:22:00.020 INFO:teuthology.packaging:ref: None 2024-03-29T04:22:00.020 INFO:teuthology.packaging:tag: None 2024-03-29T04:22:00.021 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:22:00.021 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:22:00.021 DEBUG:teuthology.orchestra.run.smithi029:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-29T04:22:00.064 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-29T04:22:00.065 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:22:00.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-29T04:22:00.091 DEBUG:teuthology.orchestra.run.smithi029:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-29T04:22:00.096 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-03-29T04:22:00.096 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:22:00.096 INFO:teuthology.packaging:ref: None 2024-03-29T04:22:00.096 INFO:teuthology.packaging:tag: None 2024-03-29T04:22:00.096 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:22:00.096 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:22:00.097 DEBUG:teuthology.orchestra.run.smithi099:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-29T04:22:00.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-29T04:22:00.172 DEBUG:teuthology.orchestra.run.smithi029:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-29T04:22:00.238 INFO:teuthology.orchestra.run.smithi029.stdout:check_obsoletes = 1 2024-03-29T04:22:00.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum clean all 2024-03-29T04:22:00.249 DEBUG:teuthology.orchestra.run.smithi099:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-29T04:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout:check_obsoletes = 1 2024-03-29T04:22:00.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum clean all 2024-03-29T04:22:00.486 INFO:teuthology.orchestra.run.smithi029.stdout:74 files removed 2024-03-29T04:22:00.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-radosgw 2024-03-29T04:22:00.581 INFO:teuthology.orchestra.run.smithi099.stdout:74 files removed 2024-03-29T04:22:00.608 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-radosgw 2024-03-29T04:22:01.221 INFO:teuthology.orchestra.run.smithi029.stdout:ceph packages for x86_64 192 kB/s | 77 kB 00:00 2024-03-29T04:22:01.365 INFO:teuthology.orchestra.run.smithi099.stdout:ceph packages for x86_64 168 kB/s | 77 kB 00:00 2024-03-29T04:22:01.605 INFO:teuthology.orchestra.run.smithi029.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2024-03-29T04:22:01.749 INFO:teuthology.orchestra.run.smithi099.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2024-03-29T04:22:02.023 INFO:teuthology.orchestra.run.smithi029.stdout:ceph source packages 4.7 kB/s | 1.8 kB 00:00 2024-03-29T04:22:02.283 INFO:teuthology.orchestra.run.smithi099.stdout:ceph source packages 3.6 kB/s | 1.8 kB 00:00 2024-03-29T04:22:02.485 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - AppStream 16 MB/s | 6.7 MB 00:00 2024-03-29T04:22:02.827 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - AppStream 13 MB/s | 6.7 MB 00:00 2024-03-29T04:22:04.011 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS-9 - BaseOS 5.5 MB/s | 2.0 MB 00:00 2024-03-29T04:22:04.186 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS-9 - BaseOS 7.8 MB/s | 2.0 MB 00:00 2024-03-29T04:22:05.279 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - BaseOS 8.6 MB/s | 8.0 MB 00:00 2024-03-29T04:22:05.434 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - BaseOS 8.7 MB/s | 8.0 MB 00:00 2024-03-29T04:22:07.451 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-03-29T04:22:11.752 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - CRB 9.6 MB/s | 6.1 MB 00:00 2024-03-29T04:22:12.668 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - AppStream 3.1 MB/s | 19 MB 00:06 2024-03-29T04:22:13.146 INFO:teuthology.orchestra.run.smithi029.stdout:CentOS Stream 9 - Extras packages 74 kB/s | 16 kB 00:00 2024-03-29T04:22:14.866 INFO:teuthology.orchestra.run.smithi029.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-03-29T04:22:16.827 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - CRB 9.0 MB/s | 6.1 MB 00:00 2024-03-29T04:22:19.053 INFO:teuthology.orchestra.run.smithi099.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 16 kB 00:01 2024-03-29T04:22:20.075 INFO:teuthology.orchestra.run.smithi029.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-03-29T04:22:20.715 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-03-29T04:22:22.865 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:22:22.867 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:22:22.867 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:22:22.867 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:22:22.867 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:22:22.867 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-radosgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 14 M 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout:Upgrading: 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: librados2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.1 M 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: librbd1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-base x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 5.0 M 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 22 M 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-selinux x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 636 k 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-29T04:22:22.868 INFO:teuthology.orchestra.run.smithi029.stdout: libradosstriper1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 458 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: librgw2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 4.9 M 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-argparse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 45 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 118 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cephfs x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 156 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rados x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 319 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 293 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 99 k 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:22:22.869 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:22:22.870 INFO:teuthology.orchestra.run.smithi029.stdout:Install 22 Packages 2024-03-29T04:22:22.870 INFO:teuthology.orchestra.run.smithi029.stdout:Upgrade 2 Packages 2024-03-29T04:22:22.870 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:22.870 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 58 M 2024-03-29T04:22:22.870 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:22:24.269 INFO:teuthology.orchestra.run.smithi029.stdout:(1/24): ceph-base-17.2.7-767.g9b4b8ffd.el9.x86_ 4.1 MB/s | 5.0 MB 00:01 2024-03-29T04:22:24.346 INFO:teuthology.orchestra.run.smithi029.stdout:(2/24): ceph-selinux-17.2.7-767.g9b4b8ffd.el9.x 328 kB/s | 25 kB 00:00 2024-03-29T04:22:24.530 INFO:teuthology.orchestra.run.smithi029.stdout:(3/24): libcephfs2-17.2.7-767.g9b4b8ffd.el9.x86 3.4 MB/s | 636 kB 00:00 2024-03-29T04:22:24.647 INFO:teuthology.orchestra.run.smithi029.stdout:(4/24): libradosstriper1-17.2.7-767.g9b4b8ffd.e 3.8 MB/s | 458 kB 00:00 2024-03-29T04:22:25.047 INFO:teuthology.orchestra.run.smithi029.stdout:(5/24): ceph-common-17.2.7-767.g9b4b8ffd.el9.x8 11 MB/s | 22 MB 00:01 2024-03-29T04:22:25.089 INFO:teuthology.orchestra.run.smithi029.stdout:(6/24): python3-ceph-argparse-17.2.7-767.g9b4b8 1.1 MB/s | 45 kB 00:00 2024-03-29T04:22:25.157 INFO:teuthology.orchestra.run.smithi029.stdout:(7/24): python3-ceph-common-17.2.7-767.g9b4b8ff 1.7 MB/s | 118 kB 00:00 2024-03-29T04:22:25.224 INFO:teuthology.orchestra.run.smithi029.stdout:(8/24): python3-cephfs-17.2.7-767.g9b4b8ffd.el9 2.3 MB/s | 156 kB 00:00 2024-03-29T04:22:25.466 INFO:teuthology.orchestra.run.smithi029.stdout:(9/24): ceph-radosgw-17.2.7-767.g9b4b8ffd.el9.x 5.9 MB/s | 14 MB 00:02 2024-03-29T04:22:25.500 INFO:teuthology.orchestra.run.smithi029.stdout:(10/24): python3-rados-17.2.7-767.g9b4b8ffd.el9 1.1 MB/s | 319 kB 00:00 2024-03-29T04:22:25.604 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-03-29T04:22:25.609 INFO:teuthology.orchestra.run.smithi029.stdout:(11/24): librgw2-17.2.7-767.g9b4b8ffd.el9.x86_6 5.1 MB/s | 4.9 MB 00:00 2024-03-29T04:22:25.634 INFO:teuthology.orchestra.run.smithi029.stdout:(12/24): python3-rgw-17.2.7-767.g9b4b8ffd.el9.x 740 kB/s | 99 kB 00:00 2024-03-29T04:22:25.693 INFO:teuthology.orchestra.run.smithi029.stdout:(13/24): python3-rbd-17.2.7-767.g9b4b8ffd.el9.x 1.3 MB/s | 293 kB 00:00 2024-03-29T04:22:25.777 INFO:teuthology.orchestra.run.smithi029.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 949 kB/s | 159 kB 00:00 2024-03-29T04:22:25.811 INFO:teuthology.orchestra.run.smithi029.stdout:(15/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 297 kB/s | 44 kB 00:00 2024-03-29T04:22:25.845 INFO:teuthology.orchestra.run.smithi029.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 5.3 MB/s | 661 kB 00:00 2024-03-29T04:22:25.879 INFO:teuthology.orchestra.run.smithi029.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.7 MB/s | 282 kB 00:00 2024-03-29T04:22:25.906 INFO:teuthology.orchestra.run.smithi029.stdout:(18/24): python3-prettytable-0.7.2-27.el9.noarc 440 kB/s | 41 kB 00:00 2024-03-29T04:22:26.389 INFO:teuthology.orchestra.run.smithi029.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 100 kB/s | 49 kB 00:00 2024-03-29T04:22:26.473 INFO:teuthology.orchestra.run.smithi029.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 119 kB/s | 67 kB 00:00 2024-03-29T04:22:26.574 INFO:teuthology.orchestra.run.smithi029.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 424 kB/s | 309 kB 00:00 2024-03-29T04:22:26.899 INFO:teuthology.orchestra.run.smithi029.stdout:(22/24): librbd1-17.2.7-767.g9b4b8ffd.el9.x86_6 9.0 MB/s | 2.9 MB 00:00 2024-03-29T04:22:26.975 INFO:teuthology.orchestra.run.smithi029.stdout:(23/24): librados2-17.2.7-767.g9b4b8ffd.el9.x86 6.1 MB/s | 3.1 MB 00:00 2024-03-29T04:22:27.025 INFO:teuthology.orchestra.run.smithi029.stdout:(24/24): thrift-0.15.0-2.el9.x86_64.rpm 2.5 MB/s | 1.6 MB 00:00 2024-03-29T04:22:27.033 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:27.033 INFO:teuthology.orchestra.run.smithi029.stdout:Total 14 MB/s | 58 MB 00:04 2024-03-29T04:22:27.033 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:22:27.071 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:22:27.072 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:22:27.542 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:22:27.543 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:22:28.203 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:22:28.204 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-radosgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 14 M 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout:Upgrading: 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: librados2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.1 M 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: librbd1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 5.0 M 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 22 M 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-selinux x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 636 k 2024-03-29T04:22:28.205 INFO:teuthology.orchestra.run.smithi099.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: libradosstriper1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 458 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: librgw2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 4.9 M 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-argparse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 45 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 118 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cephfs x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 156 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-29T04:22:28.206 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rados x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 319 k 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 293 k 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 99 k 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout:Install 22 Packages 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout:Upgrade 2 Packages 2024-03-29T04:22:28.207 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:28.208 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 58 M 2024-03-29T04:22:28.208 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:22:28.758 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:22:29.157 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-03-29T04:22:29.262 INFO:teuthology.orchestra.run.smithi029.stdout: Upgrading : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:22:29.361 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:22:29.455 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:22:29.484 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:22:29.556 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:22:29.668 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-03-29T04:22:29.706 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-03-29T04:22:29.788 INFO:teuthology.orchestra.run.smithi099.stdout:(1/24): ceph-base-17.2.7-767.g9b4b8ffd.el9.x86_ 3.6 MB/s | 5.0 MB 00:01 2024-03-29T04:22:30.005 INFO:teuthology.orchestra.run.smithi099.stdout:(2/24): ceph-common-17.2.7-767.g9b4b8ffd.el9.x8 14 MB/s | 22 MB 00:01 2024-03-29T04:22:30.021 INFO:teuthology.orchestra.run.smithi099.stdout:(3/24): ceph-selinux-17.2.7-767.g9b4b8ffd.el9.x 108 kB/s | 25 kB 00:00 2024-03-29T04:22:30.063 INFO:teuthology.orchestra.run.smithi099.stdout:(4/24): libcephfs2-17.2.7-767.g9b4b8ffd.el9.x86 11 MB/s | 636 kB 00:00 2024-03-29T04:22:30.105 INFO:teuthology.orchestra.run.smithi099.stdout:(5/24): libradosstriper1-17.2.7-767.g9b4b8ffd.e 5.4 MB/s | 458 kB 00:00 2024-03-29T04:22:30.166 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-03-29T04:22:30.205 INFO:teuthology.orchestra.run.smithi099.stdout:(6/24): python3-ceph-argparse-17.2.7-767.g9b4b8 452 kB/s | 45 kB 00:00 2024-03-29T04:22:30.206 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:22:30.271 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:22:30.300 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-03-29T04:22:30.314 INFO:teuthology.orchestra.run.smithi099.stdout:(7/24): python3-ceph-common-17.2.7-767.g9b4b8ff 1.1 MB/s | 118 kB 00:00 2024-03-29T04:22:30.352 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 10/26 2024-03-29T04:22:30.398 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:22:30.398 INFO:teuthology.orchestra.run.smithi099.stdout:(8/24): python3-cephfs-17.2.7-767.g9b4b8ffd.el9 1.8 MB/s | 156 kB 00:00 2024-03-29T04:22:30.470 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:22:30.496 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:22:30.562 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:22:30.581 INFO:teuthology.orchestra.run.smithi099.stdout:(9/24): ceph-radosgw-17.2.7-767.g9b4b8ffd.el9.x 6.4 MB/s | 14 MB 00:02 2024-03-29T04:22:30.602 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-03-29T04:22:30.641 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-03-29T04:22:30.657 INFO:teuthology.orchestra.run.smithi099.stdout:(10/24): librgw2-17.2.7-767.g9b4b8ffd.el9.x86_6 8.3 MB/s | 4.9 MB 00:00 2024-03-29T04:22:30.683 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-03-29T04:22:30.690 INFO:teuthology.orchestra.run.smithi099.stdout:(11/24): python3-rados-17.2.7-767.g9b4b8ffd.el9 1.1 MB/s | 319 kB 00:00 2024-03-29T04:22:30.715 INFO:teuthology.orchestra.run.smithi099.stdout:(12/24): python3-rbd-17.2.7-767.g9b4b8ffd.el9.x 2.1 MB/s | 293 kB 00:00 2024-03-29T04:22:30.740 INFO:teuthology.orchestra.run.smithi099.stdout:(13/24): python3-rgw-17.2.7-767.g9b4b8ffd.el9.x 1.2 MB/s | 99 kB 00:00 2024-03-29T04:22:30.866 INFO:teuthology.orchestra.run.smithi099.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 294 kB/s | 44 kB 00:00 2024-03-29T04:22:30.891 INFO:teuthology.orchestra.run.smithi099.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 792 kB/s | 159 kB 00:00 2024-03-29T04:22:30.975 INFO:teuthology.orchestra.run.smithi099.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.5 MB/s | 282 kB 00:00 2024-03-29T04:22:30.992 INFO:teuthology.orchestra.run.smithi099.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 411 kB/s | 41 kB 00:00 2024-03-29T04:22:30.993 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-03-29T04:22:31.022 INFO:teuthology.orchestra.run.smithi029.stdout: Upgrading : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:22:31.026 INFO:teuthology.orchestra.run.smithi099.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.7 MB/s | 661 kB 00:00 2024-03-29T04:22:31.119 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:22:31.134 INFO:teuthology.orchestra.run.smithi099.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 342 kB/s | 49 kB 00:00 2024-03-29T04:22:31.201 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 19/26 2024-03-29T04:22:31.202 INFO:teuthology.orchestra.run.smithi099.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 384 kB/s | 67 kB 00:00 2024-03-29T04:22:31.234 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 20/26 2024-03-29T04:22:31.277 INFO:teuthology.orchestra.run.smithi099.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.0 MB/s | 309 kB 00:00 2024-03-29T04:22:31.419 INFO:teuthology.orchestra.run.smithi099.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 5.6 MB/s | 1.6 MB 00:00 2024-03-29T04:22:31.477 INFO:teuthology.orchestra.run.smithi099.stdout:(23/24): librbd1-17.2.7-767.g9b4b8ffd.el9.x86_6 15 MB/s | 2.9 MB 00:00 2024-03-29T04:22:31.711 INFO:teuthology.orchestra.run.smithi099.stdout:(24/24): librados2-17.2.7-767.g9b4b8ffd.el9.x86 6.0 MB/s | 3.1 MB 00:00 2024-03-29T04:22:31.715 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:31.715 INFO:teuthology.orchestra.run.smithi099.stdout:Total 16 MB/s | 58 MB 00:03 2024-03-29T04:22:31.716 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:22:31.749 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:22:31.749 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:22:32.200 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:22:32.200 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:22:33.421 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:22:33.757 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-03-29T04:22:33.802 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:22:33.804 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:22:33.847 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:22:33.890 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:22:33.973 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:22:34.023 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:22:34.092 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:22:34.141 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-03-29T04:22:34.169 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-03-29T04:22:34.384 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:22:34.428 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:22:34.513 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:22:34.514 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-29T04:22:34.514 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-29T04:22:34.514 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:34.549 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:22:34.603 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-03-29T04:22:34.626 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:22:34.688 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:22:34.717 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-03-29T04:22:34.751 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 10/26 2024-03-29T04:22:34.789 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:22:34.859 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:22:34.887 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:22:34.952 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:22:34.992 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-03-29T04:22:35.028 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-03-29T04:22:35.061 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-03-29T04:22:35.356 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-03-29T04:22:35.383 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:22:35.473 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:22:35.573 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 19/26 2024-03-29T04:22:35.610 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 20/26 2024-03-29T04:22:38.068 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:22:38.107 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:22:38.621 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:22:38.660 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:22:38.751 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:22:38.751 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-29T04:22:38.752 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-29T04:22:38.752 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:38.770 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:22:46.472 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /sys 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /proc 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /mnt 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /var/tmp 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /home 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /root 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout:skipping the directory /tmp 2024-03-29T04:22:46.473 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:47.859 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:22:47.937 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:22:47.937 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:22:47.937 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-29T04:22:47.937 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:22:47.937 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:22:47.937 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:47.939 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:22:48.023 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:22:48.023 INFO:teuthology.orchestra.run.smithi029.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /sys 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /proc 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /mnt 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /var/tmp 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /home 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /root 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /tmp 2024-03-29T04:22:48.375 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:49.647 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:22:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:22:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/26 2024-03-29T04:22:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:22:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:22:49.651 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 7/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 8/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 9/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 10/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:22:49.652 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:22:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-03-29T04:22:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-03-29T04:22:49.653 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-03-29T04:22:49.654 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 25/26 2024-03-29T04:22:49.708 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:22:49.709 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:22:49.709 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-29T04:22:49.709 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:22:49.709 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:22:49.709 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:49.710 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:22:49.792 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:22:49.792 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:22:50.569 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:22:50.569 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:50.569 INFO:teuthology.orchestra.run.smithi029.stdout:Upgraded: 2024-03-29T04:22:50.569 INFO:teuthology.orchestra.run.smithi029.stdout: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.569 INFO:teuthology.orchestra.run.smithi029.stdout: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.569 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.570 INFO:teuthology.orchestra.run.smithi029.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:50.571 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:22:50.978 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-test 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 7/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 8/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 9/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 10/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:22:51.059 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-03-29T04:22:51.060 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 25/26 2024-03-29T04:22:51.547 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:31 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:22:51.706 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-test x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 48 M 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout: libcephsqlite x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 163 k 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-29T04:22:51.707 INFO:teuthology.orchestra.run.smithi029.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:Install 6 Packages 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 49 M 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 193 M 2024-03-29T04:22:51.708 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:22:51.892 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout:Upgraded: 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-29T04:22:51.893 INFO:teuthology.orchestra.run.smithi099.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-29T04:22:51.894 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:51.895 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:22:51.944 INFO:teuthology.orchestra.run.smithi029.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.1 MB/s | 217 kB 00:00 2024-03-29T04:22:51.994 INFO:teuthology.orchestra.run.smithi029.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.8 MB/s | 300 kB 00:00 2024-03-29T04:22:52.020 INFO:teuthology.orchestra.run.smithi029.stdout:(3/6): libcephsqlite-17.2.7-767.g9b4b8ffd.el9.x 594 kB/s | 163 kB 00:00 2024-03-29T04:22:52.037 INFO:teuthology.orchestra.run.smithi029.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.4 MB/s | 63 kB 00:00 2024-03-29T04:22:52.112 INFO:teuthology.orchestra.run.smithi029.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 2.0 MB/s | 187 kB 00:00 2024-03-29T04:22:52.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-test 2024-03-29T04:22:52.780 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:00:27 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:22:52.934 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-test x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 48 M 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: libcephsqlite x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 163 k 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-29T04:22:52.935 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout:Install 6 Packages 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 49 M 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 193 M 2024-03-29T04:22:52.936 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:22:53.219 INFO:teuthology.orchestra.run.smithi099.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.2 MB/s | 217 kB 00:00 2024-03-29T04:22:53.269 INFO:teuthology.orchestra.run.smithi099.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.9 MB/s | 300 kB 00:00 2024-03-29T04:22:53.319 INFO:teuthology.orchestra.run.smithi099.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 63 kB 00:00 2024-03-29T04:22:53.386 INFO:teuthology.orchestra.run.smithi099.stdout:(4/6): jq-1.6-16.el9.x86_64.rpm 2.7 MB/s | 187 kB 00:00 2024-03-29T04:22:53.828 INFO:teuthology.orchestra.run.smithi099.stdout:(5/6): libcephsqlite-17.2.7-767.g9b4b8ffd.el9.x 209 kB/s | 163 kB 00:00 2024-03-29T04:22:54.446 INFO:teuthology.orchestra.run.smithi029.stdout:(6/6): ceph-test-17.2.7-767.g9b4b8ffd.el9.x86_6 18 MB/s | 48 MB 00:02 2024-03-29T04:22:54.449 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:54.450 INFO:teuthology.orchestra.run.smithi029.stdout:Total 18 MB/s | 49 MB 00:02 2024-03-29T04:22:54.464 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:22:54.486 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:22:54.486 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:22:54.892 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:22:54.893 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:22:55.491 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:22:55.589 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-03-29T04:22:55.633 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-03-29T04:22:55.666 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:22:55.710 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-03-29T04:22:55.746 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:23:00.416 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:23:00.502 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:23:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:23:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/6 2024-03-29T04:23:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/6 2024-03-29T04:23:04.117 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:23:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-03-29T04:23:04.118 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: jq-1.6-16.el9.x86_64 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-29T04:23:04.540 INFO:teuthology.orchestra.run.smithi029.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-29T04:23:04.541 INFO:teuthology.orchestra.run.smithi029.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-29T04:23:04.541 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:04.541 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:04.862 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph 2024-03-29T04:23:05.435 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:45 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:05.610 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================== 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================== 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout: ceph x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 6.5 k 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mds x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.1 M 2024-03-29T04:23:05.613 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 1.4 M 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-modules-core noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 240 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mon x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.9 M 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-osd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 15 M 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-29T04:23:05.614 INFO:teuthology.orchestra.run.smithi029.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-29T04:23:05.615 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================== 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout:Install 33 Packages 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:05.616 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 27 M 2024-03-29T04:23:05.617 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 96 M 2024-03-29T04:23:05.617 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:06.267 INFO:teuthology.orchestra.run.smithi029.stdout:(1/33): ceph-17.2.7-767.g9b4b8ffd.el9.x86_64.rp 28 kB/s | 6.5 kB 00:00 2024-03-29T04:23:06.417 INFO:teuthology.orchestra.run.smithi029.stdout:(2/33): ceph-mds-17.2.7-767.g9b4b8ffd.el9.x86_6 5.4 MB/s | 2.1 MB 00:00 2024-03-29T04:23:06.926 INFO:teuthology.orchestra.run.smithi029.stdout:(3/33): ceph-mgr-17.2.7-767.g9b4b8ffd.el9.x86_6 1.6 MB/s | 1.4 MB 00:00 2024-03-29T04:23:07.360 INFO:teuthology.orchestra.run.smithi029.stdout:(4/33): ceph-osd-17.2.7-767.g9b4b8ffd.el9.x86_6 16 MB/s | 15 MB 00:00 2024-03-29T04:23:07.394 INFO:teuthology.orchestra.run.smithi029.stdout:(5/33): ceph-mgr-modules-core-17.2.7-767.g9b4b8 513 kB/s | 240 kB 00:00 2024-03-29T04:23:07.503 INFO:teuthology.orchestra.run.smithi029.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 381 kB/s | 32 kB 00:00 2024-03-29T04:23:07.528 INFO:teuthology.orchestra.run.smithi029.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 860 kB/s | 144 kB 00:00 2024-03-29T04:23:07.546 INFO:teuthology.orchestra.run.smithi029.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.1 MB/s | 47 kB 00:00 2024-03-29T04:23:07.564 INFO:teuthology.orchestra.run.smithi029.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 1.1 MB/s | 39 kB 00:00 2024-03-29T04:23:07.607 INFO:teuthology.orchestra.run.smithi029.stdout:(10/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.9 MB/s | 241 kB 00:00 2024-03-29T04:23:07.699 INFO:teuthology.orchestra.run.smithi029.stdout:(11/33): ceph-mon-17.2.7-767.g9b4b8ffd.el9.x86_ 2.7 MB/s | 3.9 MB 00:01 2024-03-29T04:23:07.741 INFO:teuthology.orchestra.run.smithi029.stdout:(12/33): python3-cryptography-36.0.1-4.el9.x86_ 6.5 MB/s | 1.1 MB 00:00 2024-03-29T04:23:07.775 INFO:teuthology.orchestra.run.smithi029.stdout:(13/33): python3-ply-3.11-14.el9.noarch.rpm 616 kB/s | 103 kB 00:00 2024-03-29T04:23:07.800 INFO:teuthology.orchestra.run.smithi029.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 1.2 MB/s | 124 kB 00:00 2024-03-29T04:23:07.843 INFO:teuthology.orchestra.run.smithi029.stdout:(15/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.7 MB/s | 187 kB 00:00 2024-03-29T04:23:07.968 INFO:teuthology.orchestra.run.smithi029.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 259 kB/s | 43 kB 00:00 2024-03-29T04:23:08.077 INFO:teuthology.orchestra.run.smithi029.stdout:(17/33): python3-requests-2.25.1-8.el9.noarch.r 414 kB/s | 125 kB 00:00 2024-03-29T04:23:08.119 INFO:teuthology.orchestra.run.smithi029.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 621 kB/s | 172 kB 00:00 2024-03-29T04:23:08.178 INFO:teuthology.orchestra.run.smithi029.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 106 kB/s | 11 kB 00:00 2024-03-29T04:23:08.204 INFO:teuthology.orchestra.run.smithi029.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 211 kB/s | 18 kB 00:00 2024-03-29T04:23:08.237 INFO:teuthology.orchestra.run.smithi029.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 393 kB/s | 23 kB 00:00 2024-03-29T04:23:08.254 INFO:teuthology.orchestra.run.smithi029.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 383 kB/s | 19 kB 00:00 2024-03-29T04:23:08.280 INFO:teuthology.orchestra.run.smithi029.stdout:(23/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.1 MB/s | 358 kB 00:00 2024-03-29T04:23:08.297 INFO:teuthology.orchestra.run.smithi029.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 330 kB/s | 20 kB 00:00 2024-03-29T04:23:08.324 INFO:teuthology.orchestra.run.smithi029.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 666 kB/s | 46 kB 00:00 2024-03-29T04:23:08.350 INFO:teuthology.orchestra.run.smithi029.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.1 MB/s | 79 kB 00:00 2024-03-29T04:23:08.375 INFO:teuthology.orchestra.run.smithi029.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 324 kB/s | 16 kB 00:00 2024-03-29T04:23:08.418 INFO:teuthology.orchestra.run.smithi029.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.3 MB/s | 90 kB 00:00 2024-03-29T04:23:08.443 INFO:teuthology.orchestra.run.smithi029.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 530 kB/s | 36 kB 00:00 2024-03-29T04:23:08.477 INFO:teuthology.orchestra.run.smithi029.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.5 MB/s | 272 kB 00:00 2024-03-29T04:23:08.527 INFO:teuthology.orchestra.run.smithi029.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 398 kB/s | 20 kB 00:00 2024-03-29T04:23:08.553 INFO:teuthology.orchestra.run.smithi029.stdout:(32/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.7 MB/s | 230 kB 00:00 2024-03-29T04:23:08.611 INFO:teuthology.orchestra.run.smithi029.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-03-29T04:23:08.615 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:08.615 INFO:teuthology.orchestra.run.smithi029.stdout:Total 9.0 MB/s | 27 MB 00:02 2024-03-29T04:23:08.630 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:08.673 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:08.673 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:08.751 INFO:teuthology.orchestra.run.smithi099.stdout:(6/6): ceph-test-17.2.7-767.g9b4b8ffd.el9.x86_6 3.0 MB/s | 48 MB 00:15 2024-03-29T04:23:08.754 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:08.754 INFO:teuthology.orchestra.run.smithi099.stdout:Total 3.1 MB/s | 49 MB 00:15 2024-03-29T04:23:08.768 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:08.788 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:08.789 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:09.061 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:09.061 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:09.191 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:09.192 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:09.785 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:09.868 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-03-29T04:23:09.909 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-03-29T04:23:09.942 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:23:09.984 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-03-29T04:23:10.012 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:23:10.316 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:10.374 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-03-29T04:23:10.407 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-03-29T04:23:10.449 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-03-29T04:23:10.556 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-03-29T04:23:10.633 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-03-29T04:23:10.688 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-03-29T04:23:10.773 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-03-29T04:23:10.833 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-03-29T04:23:10.879 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-03-29T04:23:10.940 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-03-29T04:23:11.014 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-03-29T04:23:11.218 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-03-29T04:23:11.288 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-03-29T04:23:11.335 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-03-29T04:23:11.381 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-03-29T04:23:11.428 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-03-29T04:23:11.509 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-03-29T04:23:11.548 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:23:11.581 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-03-29T04:23:11.623 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-03-29T04:23:11.667 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-03-29T04:23:11.703 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-03-29T04:23:11.797 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-03-29T04:23:11.852 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-03-29T04:23:11.945 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-03-29T04:23:12.095 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-03-29T04:23:12.229 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:23:12.388 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 28/33 2024-03-29T04:23:12.421 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:23:12.446 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:23:12.447 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:12.447 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-29T04:23:12.447 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:23:12.447 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:23:12.447 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:13.848 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:23:13.873 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:23:13.873 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:13.873 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-29T04:23:13.873 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:23:13.874 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:23:13.874 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:14.276 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:23:14.299 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:23:14.299 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:14.299 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-29T04:23:14.299 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:23:14.300 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:23:14.300 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:23:14.460 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:23:14.510 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:23:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:23:14.534 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:14.534 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-29T04:23:14.534 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:23:14.534 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:23:14.534 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:14.602 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:23:16.899 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:23:16.899 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/6 2024-03-29T04:23:16.899 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/6 2024-03-29T04:23:16.899 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:23:16.899 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-03-29T04:23:16.899 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-03-29T04:23:17.054 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:23:17.054 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 6/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-03-29T04:23:17.055 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-03-29T04:23:17.056 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:23:17.057 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-03-29T04:23:17.057 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-03-29T04:23:17.057 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-03-29T04:23:17.057 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-03-29T04:23:17.057 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-03-29T04:23:17.303 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-03-29T04:23:17.303 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:17.303 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:17.303 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:17.303 INFO:teuthology.orchestra.run.smithi099.stdout: jq-1.6-16.el9.x86_64 2024-03-29T04:23:17.304 INFO:teuthology.orchestra.run.smithi099.stdout: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:17.304 INFO:teuthology.orchestra.run.smithi099.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-29T04:23:17.304 INFO:teuthology.orchestra.run.smithi099.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-29T04:23:17.304 INFO:teuthology.orchestra.run.smithi099.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-29T04:23:17.304 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:17.304 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:17.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph 2024-03-29T04:23:18.002 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-03-29T04:23:18.002 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:18.002 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-29T04:23:18.003 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-ply-3.11-14.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-29T04:23:18.004 INFO:teuthology.orchestra.run.smithi029.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:18.005 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:18.169 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:00:53 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:18.332 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================== 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================== 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout: ceph x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 6.5 k 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:23:18.334 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mds x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.1 M 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 1.4 M 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 240 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mon x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.9 M 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-osd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 15 M 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-29T04:23:18.335 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-29T04:23:18.336 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================== 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout:Install 33 Packages 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:18.337 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 27 M 2024-03-29T04:23:18.338 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 96 M 2024-03-29T04:23:18.338 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:18.421 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-base 2024-03-29T04:23:18.796 INFO:teuthology.orchestra.run.smithi099.stdout:(1/33): ceph-17.2.7-767.g9b4b8ffd.el9.x86_64.rp 33 kB/s | 6.5 kB 00:00 2024-03-29T04:23:18.985 INFO:teuthology.orchestra.run.smithi099.stdout:(2/33): ceph-mds-17.2.7-767.g9b4b8ffd.el9.x86_6 5.3 MB/s | 2.1 MB 00:00 2024-03-29T04:23:18.997 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:59 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:19.097 INFO:teuthology.orchestra.run.smithi029.stdout:Package ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:23:19.152 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:19.153 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:23:19.153 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:19.219 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install cephadm 2024-03-29T04:23:19.419 INFO:teuthology.orchestra.run.smithi099.stdout:(3/33): ceph-mgr-17.2.7-767.g9b4b8ffd.el9.x86_6 1.7 MB/s | 1.4 MB 00:00 2024-03-29T04:23:19.602 INFO:teuthology.orchestra.run.smithi099.stdout:(4/33): ceph-mgr-modules-core-17.2.7-767.g9b4b8 1.3 MB/s | 240 kB 00:00 2024-03-29T04:23:19.728 INFO:teuthology.orchestra.run.smithi099.stdout:(5/33): ceph-mon-17.2.7-767.g9b4b8ffd.el9.x86_6 4.2 MB/s | 3.9 MB 00:00 2024-03-29T04:23:19.788 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:00:59 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:19.886 INFO:teuthology.orchestra.run.smithi099.stdout:(6/33): ceph-osd-17.2.7-767.g9b4b8ffd.el9.x86_6 17 MB/s | 15 MB 00:00 2024-03-29T04:23:19.903 INFO:teuthology.orchestra.run.smithi099.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 480 kB/s | 144 kB 00:00 2024-03-29T04:23:19.928 INFO:teuthology.orchestra.run.smithi099.stdout:(8/33): python3-markupsafe-1.1.1-12.el9.x86_64. 157 kB/s | 32 kB 00:00 2024-03-29T04:23:19.943 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout: cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 82 k 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:23:19.944 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:19.945 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 82 k 2024-03-29T04:23:19.945 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 372 k 2024-03-29T04:23:19.945 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:19.953 INFO:teuthology.orchestra.run.smithi099.stdout:(9/33): python3-pytz-2021.1-5.el9.noarch.rpm 709 kB/s | 47 kB 00:00 2024-03-29T04:23:19.979 INFO:teuthology.orchestra.run.smithi099.stdout:(10/33): python3-toml-0.10.2-6.el9.noarch.rpm 511 kB/s | 39 kB 00:00 2024-03-29T04:23:20.012 INFO:teuthology.orchestra.run.smithi099.stdout:(11/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.8 MB/s | 241 kB 00:00 2024-03-29T04:23:20.037 INFO:teuthology.orchestra.run.smithi099.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 1.7 MB/s | 103 kB 00:00 2024-03-29T04:23:20.121 INFO:teuthology.orchestra.run.smithi099.stdout:(13/33): python3-cryptography-36.0.1-4.el9.x86_ 8.1 MB/s | 1.1 MB 00:00 2024-03-29T04:23:20.146 INFO:teuthology.orchestra.run.smithi099.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 928 kB/s | 124 kB 00:00 2024-03-29T04:23:20.172 INFO:teuthology.orchestra.run.smithi099.stdout:(15/33): python3-urllib3-1.26.5-5.el9.noarch.rp 1.4 MB/s | 187 kB 00:00 2024-03-29T04:23:20.247 INFO:teuthology.orchestra.run.smithi029.stdout:cephadm-17.2.7-767.g9b4b8ffd.el9.noarch.rpm 271 kB/s | 82 kB 00:00 2024-03-29T04:23:20.248 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:20.248 INFO:teuthology.orchestra.run.smithi029.stdout:Total 270 kB/s | 82 kB 00:00 2024-03-29T04:23:20.248 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:20.254 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:20.254 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:20.256 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:20.256 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:20.338 INFO:teuthology.orchestra.run.smithi099.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 577 kB/s | 125 kB 00:00 2024-03-29T04:23:20.372 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:20.464 INFO:teuthology.orchestra.run.smithi099.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 149 kB/s | 43 kB 00:00 2024-03-29T04:23:20.539 INFO:teuthology.orchestra.run.smithi099.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 143 kB/s | 11 kB 00:00 2024-03-29T04:23:20.597 INFO:teuthology.orchestra.run.smithi099.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 404 kB/s | 172 kB 00:00 2024-03-29T04:23:20.614 INFO:teuthology.orchestra.run.smithi099.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 235 kB/s | 18 kB 00:00 2024-03-29T04:23:20.665 INFO:teuthology.orchestra.run.smithi099.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 346 kB/s | 23 kB 00:00 2024-03-29T04:23:20.690 INFO:teuthology.orchestra.run.smithi099.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 258 kB/s | 19 kB 00:00 2024-03-29T04:23:20.741 INFO:teuthology.orchestra.run.smithi099.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-03-29T04:23:20.774 INFO:teuthology.orchestra.run.smithi099.stdout:(24/33): python3-cherrypy-18.6.1-2.el9.noarch.r 825 kB/s | 358 kB 00:00 2024-03-29T04:23:20.791 INFO:teuthology.orchestra.run.smithi099.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 460 kB/s | 46 kB 00:00 2024-03-29T04:23:20.825 INFO:teuthology.orchestra.run.smithi099.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 942 kB/s | 79 kB 00:00 2024-03-29T04:23:20.867 INFO:teuthology.orchestra.run.smithi099.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-03-29T04:23:20.869 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:20.883 INFO:teuthology.orchestra.run.smithi099.stdout:(28/33): python3-portend-3.1.0-2.el9.noarch.rpm 178 kB/s | 16 kB 00:00 2024-03-29T04:23:20.917 INFO:teuthology.orchestra.run.smithi099.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 977 kB/s | 90 kB 00:00 2024-03-29T04:23:20.950 INFO:teuthology.orchestra.run.smithi099.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 427 kB/s | 36 kB 00:00 2024-03-29T04:23:20.976 INFO:teuthology.orchestra.run.smithi099.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.4 MB/s | 230 kB 00:00 2024-03-29T04:23:21.003 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:21.026 INFO:teuthology.orchestra.run.smithi099.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 266 kB/s | 20 kB 00:00 2024-03-29T04:23:21.076 INFO:teuthology.orchestra.run.smithi099.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.6 MB/s | 427 kB 00:00 2024-03-29T04:23:21.082 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:21.083 INFO:teuthology.orchestra.run.smithi099.stdout:Total 9.8 MB/s | 27 MB 00:02 2024-03-29T04:23:21.097 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:21.137 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:21.137 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:21.412 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:21.473 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:21.474 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:21.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:21.701 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:21.701 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:21.701 INFO:teuthology.orchestra.run.smithi029.stdout: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:21.701 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:21.701 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:21.904 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-immutable-object-cache 2024-03-29T04:23:22.478 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:02 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:22.635 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:22.635 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 137 k 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:23:22.636 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:22.637 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 137 k 2024-03-29T04:23:22.637 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 379 k 2024-03-29T04:23:22.637 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:22.779 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-03-29T04:23:22.811 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-03-29T04:23:22.844 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-03-29T04:23:22.939 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-03-29T04:23:23.022 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-03-29T04:23:23.023 INFO:teuthology.orchestra.run.smithi029.stdout:ceph-immutable-object-cache-17.2.7-767.g9b4b8ff 356 kB/s | 137 kB 00:00 2024-03-29T04:23:23.024 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:23.024 INFO:teuthology.orchestra.run.smithi029.stdout:Total 354 kB/s | 137 kB 00:00 2024-03-29T04:23:23.024 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:23.033 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:23.033 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:23.072 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-03-29T04:23:23.087 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:23.088 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:23.143 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-03-29T04:23:23.197 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-03-29T04:23:23.244 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-03-29T04:23:23.252 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:23.301 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-03-29T04:23:23.342 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:23:23.361 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-03-29T04:23:23.367 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:23:23.367 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:23.367 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-29T04:23:23.367 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:23.552 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-03-29T04:23:23.606 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-03-29T04:23:23.651 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-03-29T04:23:23.687 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-03-29T04:23:23.718 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-03-29T04:23:23.794 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-03-29T04:23:23.828 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:23:23.861 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-03-29T04:23:23.901 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-03-29T04:23:23.937 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-03-29T04:23:23.971 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-03-29T04:23:24.060 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-03-29T04:23:24.102 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-03-29T04:23:24.174 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-03-29T04:23:24.311 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-03-29T04:23:24.338 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:23:24.338 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:24.338 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:24.338 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:24.338 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:24.339 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:24.436 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:23:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 28/33 2024-03-29T04:23:24.600 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr 2024-03-29T04:23:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:23:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:23:24.648 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:24.648 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-29T04:23:24.648 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:23:24.648 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:23:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:25.171 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:05 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:25.272 INFO:teuthology.orchestra.run.smithi029.stdout:Package ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:23:25.325 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:25.326 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:23:25.326 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:25.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-dashboard 2024-03-29T04:23:25.963 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:05 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:25.965 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:23:25.992 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:23:25.992 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:25.992 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-29T04:23:25.992 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:23:25.992 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:23:25.992 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:26.119 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:26.120 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:26.120 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:26.120 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:26.120 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:26.120 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-dashboard noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 1.7 M 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-grafana-dashboards noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 23 k 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-prometheus-alerts noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 15 k 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:Installing weak dependencies: 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout:Install 7 Packages 2024-03-29T04:23:26.121 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:26.122 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 2.0 M 2024-03-29T04:23:26.122 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 8.1 M 2024-03-29T04:23:26.122 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:26.369 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:23:26.393 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:23:26.393 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:26.393 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-29T04:23:26.393 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:23:26.393 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:23:26.393 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:26.461 INFO:teuthology.orchestra.run.smithi029.stdout:(1/7): ceph-prometheus-alerts-17.2.7-767.g9b4b8 57 kB/s | 15 kB 00:00 2024-03-29T04:23:26.486 INFO:teuthology.orchestra.run.smithi029.stdout:(2/7): ceph-grafana-dashboards-17.2.7-767.g9b4b 82 kB/s | 23 kB 00:00 2024-03-29T04:23:26.587 INFO:teuthology.orchestra.run.smithi029.stdout:(3/7): ceph-mgr-dashboard-17.2.7-767.g9b4b8ffd. 4.4 MB/s | 1.7 MB 00:00 2024-03-29T04:23:26.592 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:23:26.616 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:23:26.616 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:26.616 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-29T04:23:26.616 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:23:26.616 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:23:26.616 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:26.681 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:23:26.796 INFO:teuthology.orchestra.run.smithi029.stdout:(4/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 27 kB/s | 9.0 kB 00:00 2024-03-29T04:23:26.980 INFO:teuthology.orchestra.run.smithi029.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 83 kB/s | 41 kB 00:00 2024-03-29T04:23:27.005 INFO:teuthology.orchestra.run.smithi029.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 83 kB/s | 31 kB 00:00 2024-03-29T04:23:27.189 INFO:teuthology.orchestra.run.smithi029.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 480 kB/s | 188 kB 00:00 2024-03-29T04:23:27.193 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:27.193 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.9 MB/s | 2.0 MB 00:01 2024-03-29T04:23:27.193 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:27.202 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:27.202 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:27.267 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:27.267 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:27.571 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:27.641 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-03-29T04:23:27.684 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-03-29T04:23:27.740 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-03-29T04:23:27.771 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-03-29T04:23:27.801 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 5/7 2024-03-29T04:23:28.051 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 6/7 2024-03-29T04:23:28.089 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/33 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/33 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/33 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/33 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/33 2024-03-29T04:23:28.386 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 6/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-03-29T04:23:28.387 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-03-29T04:23:28.388 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-03-29T04:23:28.699 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:23:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 1/7 2024-03-29T04:23:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2/7 2024-03-29T04:23:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 3/7 2024-03-29T04:23:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-03-29T04:23:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-03-29T04:23:28.700 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-03-29T04:23:28.998 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:28.999 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:29.235 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-29T04:23:29.236 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ply-3.11-14.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-29T04:23:29.237 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:29.238 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:29.281 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-diskprediction-local 2024-03-29T04:23:29.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-base 2024-03-29T04:23:29.847 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:09 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:30.005 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout:=========================================================================================== 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout:=========================================================================================== 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 7.4 M 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:23:30.006 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-29T04:23:30.007 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:30.008 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:30.008 INFO:teuthology.orchestra.run.smithi029.stdout:=========================================================================================== 2024-03-29T04:23:30.008 INFO:teuthology.orchestra.run.smithi029.stdout:Install 12 Packages 2024-03-29T04:23:30.008 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:30.008 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 37 M 2024-03-29T04:23:30.009 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 209 M 2024-03-29T04:23:30.009 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:30.190 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:05 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:30.272 INFO:teuthology.orchestra.run.smithi029.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 224 kB/s | 32 kB 00:00 2024-03-29T04:23:30.286 INFO:teuthology.orchestra.run.smithi099.stdout:Package ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:23:30.314 INFO:teuthology.orchestra.run.smithi029.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 414 kB/s | 17 kB 00:00 2024-03-29T04:23:30.336 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:30.337 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:23:30.337 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:30.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install cephadm 2024-03-29T04:23:30.406 INFO:teuthology.orchestra.run.smithi029.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 11 MB/s | 3.0 MB 00:00 2024-03-29T04:23:30.432 INFO:teuthology.orchestra.run.smithi029.stdout:(4/12): openblas-0.3.21-2.el9.x86_64.rpm 299 kB/s | 34 kB 00:00 2024-03-29T04:23:30.641 INFO:teuthology.orchestra.run.smithi029.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2024-03-29T04:23:30.741 INFO:teuthology.orchestra.run.smithi029.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 15 MB/s | 4.8 MB 00:00 2024-03-29T04:23:30.775 INFO:teuthology.orchestra.run.smithi029.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 1.6 MB/s | 217 kB 00:00 2024-03-29T04:23:30.957 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:05 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:30.976 INFO:teuthology.orchestra.run.smithi029.stdout:(8/12): ceph-mgr-diskprediction-local-17.2.7-76 8.7 MB/s | 7.4 MB 00:00 2024-03-29T04:23:31.105 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout: cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 82 k 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 82 k 2024-03-29T04:23:31.106 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 372 k 2024-03-29T04:23:31.107 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:31.331 INFO:teuthology.orchestra.run.smithi099.stdout:cephadm-17.2.7-767.g9b4b8ffd.el9.noarch.rpm 365 kB/s | 82 kB 00:00 2024-03-29T04:23:31.332 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:31.332 INFO:teuthology.orchestra.run.smithi099.stdout:Total 363 kB/s | 82 kB 00:00 2024-03-29T04:23:31.332 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:31.337 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:31.337 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:31.339 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:31.339 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:31.351 INFO:teuthology.orchestra.run.smithi029.stdout:(9/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 26 MB/s | 16 MB 00:00 2024-03-29T04:23:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:31.443 INFO:teuthology.orchestra.run.smithi029.stdout:(10/12): python3-devel-3.9.17-2.el9.x86_64.rpm 2.6 MB/s | 245 kB 00:00 2024-03-29T04:23:31.777 INFO:teuthology.orchestra.run.smithi029.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 245 kB/s | 195 kB 00:00 2024-03-29T04:23:31.803 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:31.894 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:31.944 INFO:teuthology.orchestra.run.smithi029.stdout:(12/12): libgfortran-11.4.1-3.el9.x86_64.rpm 688 kB/s | 803 kB 00:01 2024-03-29T04:23:31.946 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:31.947 INFO:teuthology.orchestra.run.smithi029.stdout:Total 19 MB/s | 37 MB 00:01 2024-03-29T04:23:31.992 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:32.018 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:32.018 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:32.232 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:32.489 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:32.490 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:32.499 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:23:32.499 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:32.500 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:32.500 INFO:teuthology.orchestra.run.smithi099.stdout: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:32.500 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:32.500 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:32.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-immutable-object-cache 2024-03-29T04:23:33.210 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:33.274 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:08 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:33.294 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-03-29T04:23:33.334 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-03-29T04:23:33.426 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:33.426 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 137 k 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 137 k 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 379 k 2024-03-29T04:23:33.427 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:33.443 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:23:33.490 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-03-29T04:23:33.653 INFO:teuthology.orchestra.run.smithi099.stdout:ceph-immutable-object-cache-17.2.7-767.g9b4b8ff 610 kB/s | 137 kB 00:00 2024-03-29T04:23:33.654 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:33.654 INFO:teuthology.orchestra.run.smithi099.stdout:Total 606 kB/s | 137 kB 00:00 2024-03-29T04:23:33.654 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:33.661 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:33.662 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:33.688 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:23:33.751 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:33.752 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:33.839 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:23:33.866 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:33.921 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:23:33.931 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-03-29T04:23:33.946 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:23:33.946 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:23:33.946 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-29T04:23:33.946 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:34.638 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-03-29T04:23:34.766 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-03-29T04:23:34.883 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:23:34.883 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:34.884 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:34.884 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:34.884 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:34.884 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:35.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr 2024-03-29T04:23:35.684 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:10 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:35.787 INFO:teuthology.orchestra.run.smithi099.stdout:Package ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:23:35.839 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:35.840 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:23:35.840 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:35.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-dashboard 2024-03-29T04:23:36.129 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-03-29T04:23:36.460 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:11 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:36.608 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:36.609 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:36.609 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:23:36.609 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:36.609 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:36.609 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 1.7 M 2024-03-29T04:23:36.609 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-grafana-dashboards noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 23 k 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-prometheus-alerts noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 15 k 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout:Installing weak dependencies: 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout:Install 7 Packages 2024-03-29T04:23:36.610 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:36.611 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 2.0 M 2024-03-29T04:23:36.611 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 8.1 M 2024-03-29T04:23:36.611 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:36.848 INFO:teuthology.orchestra.run.smithi099.stdout:(1/7): ceph-grafana-dashboards-17.2.7-767.g9b4b 123 kB/s | 23 kB 00:00 2024-03-29T04:23:36.924 INFO:teuthology.orchestra.run.smithi099.stdout:(2/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 120 kB/s | 9.0 kB 00:00 2024-03-29T04:23:36.949 INFO:teuthology.orchestra.run.smithi099.stdout:(3/7): ceph-prometheus-alerts-17.2.7-767.g9b4b8 50 kB/s | 15 kB 00:00 2024-03-29T04:23:36.983 INFO:teuthology.orchestra.run.smithi099.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 697 kB/s | 41 kB 00:00 2024-03-29T04:23:37.025 INFO:teuthology.orchestra.run.smithi099.stdout:(5/7): ceph-mgr-dashboard-17.2.7-767.g9b4b8ffd. 4.6 MB/s | 1.7 MB 00:00 2024-03-29T04:23:37.050 INFO:teuthology.orchestra.run.smithi099.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 306 kB/s | 31 kB 00:00 2024-03-29T04:23:37.075 INFO:teuthology.orchestra.run.smithi099.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 2.0 MB/s | 188 kB 00:00 2024-03-29T04:23:37.076 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:37.077 INFO:teuthology.orchestra.run.smithi099.stdout:Total 4.3 MB/s | 2.0 MB 00:00 2024-03-29T04:23:37.077 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:37.085 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:37.085 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:37.149 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:37.149 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:37.245 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-03-29T04:23:37.280 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:23:37.469 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:37.543 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-03-29T04:23:37.577 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-03-29T04:23:37.623 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-03-29T04:23:37.655 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-03-29T04:23:37.684 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 5/7 2024-03-29T04:23:37.907 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 6/7 2024-03-29T04:23:37.936 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 1/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 3/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-03-29T04:23:38.451 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-29T04:23:38.742 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-29T04:23:38.743 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:38.743 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:38.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-diskprediction-local 2024-03-29T04:23:39.519 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:14 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:39.669 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout:=========================================================================================== 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout:=========================================================================================== 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 7.4 M 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-29T04:23:39.670 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout:=========================================================================================== 2024-03-29T04:23:39.671 INFO:teuthology.orchestra.run.smithi099.stdout:Install 12 Packages 2024-03-29T04:23:39.672 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:39.672 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 37 M 2024-03-29T04:23:39.672 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 209 M 2024-03-29T04:23:39.672 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:40.006 INFO:teuthology.orchestra.run.smithi099.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 194 kB/s | 32 kB 00:00 2024-03-29T04:23:40.048 INFO:teuthology.orchestra.run.smithi099.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 423 kB/s | 17 kB 00:00 2024-03-29T04:23:40.123 INFO:teuthology.orchestra.run.smithi099.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 11 MB/s | 3.0 MB 00:00 2024-03-29T04:23:40.149 INFO:teuthology.orchestra.run.smithi099.stdout:(4/12): openblas-0.3.21-2.el9.x86_64.rpm 344 kB/s | 34 kB 00:00 2024-03-29T04:23:40.474 INFO:teuthology.orchestra.run.smithi099.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 14 MB/s | 4.7 MB 00:00 2024-03-29T04:23:40.541 INFO:teuthology.orchestra.run.smithi099.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 12 MB/s | 4.8 MB 00:00 2024-03-29T04:23:40.566 INFO:teuthology.orchestra.run.smithi099.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.3 MB/s | 217 kB 00:00 2024-03-29T04:23:41.100 INFO:teuthology.orchestra.run.smithi099.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 28 MB/s | 16 MB 00:00 2024-03-29T04:23:41.133 INFO:teuthology.orchestra.run.smithi099.stdout:(9/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.4 MB/s | 803 kB 00:00 2024-03-29T04:23:41.183 INFO:teuthology.orchestra.run.smithi099.stdout:(10/12): python3-devel-3.9.17-2.el9.x86_64.rpm 4.8 MB/s | 245 kB 00:00 2024-03-29T04:23:41.267 INFO:teuthology.orchestra.run.smithi099.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.1 MB/s | 195 kB 00:00 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 1/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-03-29T04:23:41.318 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-03-29T04:23:41.319 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-29T04:23:41.796 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-29T04:23:41.797 INFO:teuthology.orchestra.run.smithi029.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-29T04:23:41.797 INFO:teuthology.orchestra.run.smithi029.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-29T04:23:41.797 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:41.797 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:42.277 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-rook 2024-03-29T04:23:42.684 INFO:teuthology.orchestra.run.smithi099.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-7 2.6 MB/s | 7.4 MB 00:02 2024-03-29T04:23:42.686 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:42.686 INFO:teuthology.orchestra.run.smithi099.stdout:Total 12 MB/s | 37 MB 00:03 2024-03-29T04:23:42.730 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:42.753 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:42.754 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:42.856 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:22 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:43.014 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:43.015 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-03-29T04:23:43.015 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-rook noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 47 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-29T04:23:43.016 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout:Install 13 Packages 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:43.017 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 2.0 M 2024-03-29T04:23:43.018 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 26 M 2024-03-29T04:23:43.018 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:43.206 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:43.207 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:43.229 INFO:teuthology.orchestra.run.smithi029.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 157 kB/s | 24 kB 00:00 2024-03-29T04:23:43.255 INFO:teuthology.orchestra.run.smithi029.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 100 kB/s | 18 kB 00:00 2024-03-29T04:23:43.280 INFO:teuthology.orchestra.run.smithi029.stdout:(3/13): ceph-mgr-rook-17.2.7-767.g9b4b8ffd.el9. 229 kB/s | 47 kB 00:00 2024-03-29T04:23:43.314 INFO:teuthology.orchestra.run.smithi029.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.0 MB/s | 174 kB 00:00 2024-03-29T04:23:43.340 INFO:teuthology.orchestra.run.smithi029.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 132 kB 00:00 2024-03-29T04:23:43.374 INFO:teuthology.orchestra.run.smithi029.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.2 MB/s | 211 kB 00:00 2024-03-29T04:23:43.399 INFO:teuthology.orchestra.run.smithi029.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 507 kB/s | 43 kB 00:00 2024-03-29T04:23:43.658 INFO:teuthology.orchestra.run.smithi029.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 58 kB/s | 15 kB 00:00 2024-03-29T04:23:43.683 INFO:teuthology.orchestra.run.smithi029.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 94 kB/s | 32 kB 00:00 2024-03-29T04:23:43.929 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:43.942 INFO:teuthology.orchestra.run.smithi029.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 228 kB/s | 59 kB 00:00 2024-03-29T04:23:43.968 INFO:teuthology.orchestra.run.smithi029.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 357 kB/s | 203 kB 00:00 2024-03-29T04:23:43.989 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-03-29T04:23:44.021 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-03-29T04:23:44.113 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:23:44.157 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-03-29T04:23:44.193 INFO:teuthology.orchestra.run.smithi029.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 357 kB/s | 90 kB 00:00 2024-03-29T04:23:44.244 INFO:teuthology.orchestra.run.smithi029.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 1.7 MB/s | 1.0 MB 00:00 2024-03-29T04:23:44.246 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:44.263 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:44.263 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:44.332 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:23:44.364 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:23:44.365 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:44.365 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:44.458 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-03-29T04:23:44.899 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:45.084 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-03-29T04:23:45.091 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-03-29T04:23:45.141 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-03-29T04:23:45.201 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-03-29T04:23:45.206 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-03-29T04:23:45.251 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-03-29T04:23:45.293 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-03-29T04:23:45.392 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-03-29T04:23:45.510 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-03-29T04:23:45.565 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-03-29T04:23:46.339 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-03-29T04:23:46.400 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-03-29T04:23:46.435 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-03-29T04:23:46.452 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-03-29T04:23:46.490 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-03-29T04:23:46.522 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:23:47.489 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-03-29T04:23:47.521 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/13 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-03-29T04:23:47.619 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-03-29T04:23:47.620 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-03-29T04:23:47.620 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-03-29T04:23:47.620 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-03-29T04:23:47.620 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-03-29T04:23:47.620 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-03-29T04:23:47.620 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-03-29T04:23:47.980 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-03-29T04:23:47.980 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:47.980 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:47.980 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:47.980 INFO:teuthology.orchestra.run.smithi029.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:47.981 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:48.274 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-mgr-cephadm 2024-03-29T04:23:48.851 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:28 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:49.010 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:49.011 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-03-29T04:23:49.011 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:49.011 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-03-29T04:23:49.011 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:49.011 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 123 k 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout:======================================================================================= 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout:Install 6 Packages 2024-03-29T04:23:49.012 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:49.013 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 6.8 M 2024-03-29T04:23:49.013 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 33 M 2024-03-29T04:23:49.013 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:49.403 INFO:teuthology.orchestra.run.smithi029.stdout:(1/6): ceph-mgr-cephadm-17.2.7-767.g9b4b8ffd.el 557 kB/s | 123 kB 00:00 2024-03-29T04:23:49.545 INFO:teuthology.orchestra.run.smithi029.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 16 MB/s | 5.8 MB 00:00 2024-03-29T04:23:49.812 INFO:teuthology.orchestra.run.smithi029.stdout:(3/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 393 kB/s | 248 kB 00:00 2024-03-29T04:23:49.838 INFO:teuthology.orchestra.run.smithi029.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 215 kB/s | 58 kB 00:00 2024-03-29T04:23:49.946 INFO:teuthology.orchestra.run.smithi029.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.0 MB/s | 548 kB 00:00 2024-03-29T04:23:50.079 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:23:50.079 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 1/12 2024-03-29T04:23:50.079 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-03-29T04:23:50.079 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:23:50.079 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-03-29T04:23:50.080 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-03-29T04:23:50.105 INFO:teuthology.orchestra.run.smithi029.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 174 kB/s | 51 kB 00:00 2024-03-29T04:23:50.108 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:50.108 INFO:teuthology.orchestra.run.smithi029.stdout:Total 6.2 MB/s | 6.8 MB 00:01 2024-03-29T04:23:50.121 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:50.133 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:50.133 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:50.225 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:50.225 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-29T04:23:50.483 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-29T04:23:50.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-29T04:23:50.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-29T04:23:50.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-29T04:23:50.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-29T04:23:50.484 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:50.484 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:50.528 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:50.714 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-03-29T04:23:50.768 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-03-29T04:23:50.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-rook 2024-03-29T04:23:51.323 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-03-29T04:23:51.418 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-03-29T04:23:51.419 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:26 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:51.480 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-03-29T04:23:51.509 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:23:51.573 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================= 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================= 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-rook noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 47 k 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-29T04:23:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-03-29T04:23:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-29T04:23:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-29T04:23:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-29T04:23:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-29T04:23:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================= 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout:Install 13 Packages 2024-03-29T04:23:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:51.577 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 2.0 M 2024-03-29T04:23:51.577 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 26 M 2024-03-29T04:23:51.577 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:51.896 INFO:teuthology.orchestra.run.smithi099.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 194 kB/s | 24 kB 00:00 2024-03-29T04:23:51.922 INFO:teuthology.orchestra.run.smithi099.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 120 kB/s | 18 kB 00:00 2024-03-29T04:23:51.964 INFO:teuthology.orchestra.run.smithi099.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.6 MB/s | 174 kB 00:00 2024-03-29T04:23:51.989 INFO:teuthology.orchestra.run.smithi099.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.9 MB/s | 132 kB 00:00 2024-03-29T04:23:52.014 INFO:teuthology.orchestra.run.smithi099.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-03-29T04:23:52.040 INFO:teuthology.orchestra.run.smithi099.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 853 kB/s | 43 kB 00:00 2024-03-29T04:23:52.073 INFO:teuthology.orchestra.run.smithi099.stdout:(7/13): ceph-mgr-rook-17.2.7-767.g9b4b8ffd.el9. 157 kB/s | 47 kB 00:00 2024-03-29T04:23:52.132 INFO:teuthology.orchestra.run.smithi099.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 163 kB/s | 15 kB 00:00 2024-03-29T04:23:52.157 INFO:teuthology.orchestra.run.smithi099.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 226 kB/s | 32 kB 00:00 2024-03-29T04:23:52.224 INFO:teuthology.orchestra.run.smithi099.stdout:(10/13): python3-google-auth-2.28.2-1.el9.noarc 1.3 MB/s | 203 kB 00:00 2024-03-29T04:23:52.249 INFO:teuthology.orchestra.run.smithi099.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 643 kB/s | 59 kB 00:00 2024-03-29T04:23:52.274 INFO:teuthology.orchestra.run.smithi099.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-03-29T04:23:52.350 INFO:teuthology.orchestra.run.smithi099.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 4.7 MB/s | 1.0 MB 00:00 2024-03-29T04:23:52.351 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:52.352 INFO:teuthology.orchestra.run.smithi099.stdout:Total 2.6 MB/s | 2.0 MB 00:00 2024-03-29T04:23:52.352 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:52.365 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:52.365 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:52.453 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:52.454 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:52.585 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:23:52.586 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/6 2024-03-29T04:23:52.586 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-03-29T04:23:52.586 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-03-29T04:23:52.586 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-03-29T04:23:52.586 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-29T04:23:52.877 INFO:teuthology.orchestra.run.smithi029.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-29T04:23:52.878 INFO:teuthology.orchestra.run.smithi029.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-29T04:23:52.878 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:52.878 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:52.974 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:53.137 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-03-29T04:23:53.191 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install ceph-fuse 2024-03-29T04:23:53.192 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-03-29T04:23:53.248 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-03-29T04:23:53.285 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-03-29T04:23:53.329 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-03-29T04:23:53.419 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-03-29T04:23:53.523 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-03-29T04:23:53.586 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-03-29T04:23:53.764 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:33 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:53.921 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:53.921 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repository Size 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 812 k 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout:Installing dependencies: 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout:Install 2 Packages 2024-03-29T04:23:53.922 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:53.923 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 891 k 2024-03-29T04:23:53.923 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 2.6 M 2024-03-29T04:23:53.923 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:54.053 INFO:teuthology.orchestra.run.smithi029.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 607 kB/s | 79 kB 00:00 2024-03-29T04:23:54.311 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-03-29T04:23:54.366 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-03-29T04:23:54.400 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-03-29T04:23:54.446 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-03-29T04:23:54.461 INFO:teuthology.orchestra.run.smithi029.stdout:(2/2): ceph-fuse-17.2.7-767.g9b4b8ffd.el9.x86_6 1.5 MB/s | 812 kB 00:00 2024-03-29T04:23:54.462 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:54.463 INFO:teuthology.orchestra.run.smithi029.stdout:Total 1.6 MB/s | 891 kB 00:00 2024-03-29T04:23:54.463 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:54.469 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:23:54.485 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:54.485 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:54.561 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:54.562 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:54.744 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:54.888 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-03-29T04:23:54.959 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-03-29T04:23:55.195 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-03-29T04:23:55.196 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-03-29T04:23:55.545 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-03-29T04:23:55.545 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:55.545 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:55.545 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:55.546 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:55.714 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:23:55.714 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/2 2024-03-29T04:23:55.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-cephadm 2024-03-29T04:23:55.996 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-03-29T04:23:55.997 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:55.997 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:55.997 INFO:teuthology.orchestra.run.smithi029.stdout: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-03-29T04:23:55.997 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:55.997 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:56.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librados-devel 2024-03-29T04:23:56.332 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:31 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:23:56.483 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================= 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================= 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 123 k 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-29T04:23:56.484 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-29T04:23:56.485 INFO:teuthology.orchestra.run.smithi099.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-29T04:23:56.485 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:56.485 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:23:56.485 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================= 2024-03-29T04:23:56.485 INFO:teuthology.orchestra.run.smithi099.stdout:Install 6 Packages 2024-03-29T04:23:56.485 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:56.486 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 6.8 M 2024-03-29T04:23:56.486 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 33 M 2024-03-29T04:23:56.486 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:23:56.892 INFO:teuthology.orchestra.run.smithi099.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 5.8 MB 00:00 2024-03-29T04:23:56.899 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:36 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:56.959 INFO:teuthology.orchestra.run.smithi099.stdout:(2/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 717 kB/s | 248 kB 00:00 2024-03-29T04:23:56.984 INFO:teuthology.orchestra.run.smithi099.stdout:(3/6): ceph-mgr-cephadm-17.2.7-767.g9b4b8ffd.el 332 kB/s | 123 kB 00:00 2024-03-29T04:23:57.055 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout: librados-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 126 k 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 126 k 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 448 k 2024-03-29T04:23:57.056 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:23:57.118 INFO:teuthology.orchestra.run.smithi099.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 363 kB/s | 58 kB 00:00 2024-03-29T04:23:57.135 INFO:teuthology.orchestra.run.smithi099.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 339 kB/s | 51 kB 00:00 2024-03-29T04:23:57.168 INFO:teuthology.orchestra.run.smithi099.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.9 MB/s | 548 kB 00:00 2024-03-29T04:23:57.170 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:57.170 INFO:teuthology.orchestra.run.smithi099.stdout:Total 9.9 MB/s | 6.8 MB 00:00 2024-03-29T04:23:57.183 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:23:57.194 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:23:57.194 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:23:57.281 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:23:57.281 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:23:57.284 INFO:teuthology.orchestra.run.smithi029.stdout:librados-devel-17.2.7-767.g9b4b8ffd.el9.x86_64. 554 kB/s | 126 kB 00:00 2024-03-29T04:23:57.285 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:23:57.285 INFO:teuthology.orchestra.run.smithi029.stdout:Total 550 kB/s | 126 kB 00:00 2024-03-29T04:23:57.285 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:23:57.291 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:23:57.291 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:23:57.348 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:23:57.348 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:23:57.498 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:23:57.586 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:23:57.612 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:23:57.758 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-03-29T04:23:57.803 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-03-29T04:23:58.080 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:23:58.317 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-03-29T04:23:58.368 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:23:58.368 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:58.368 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:23:58.368 INFO:teuthology.orchestra.run.smithi029.stdout: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:23:58.368 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:23:58.368 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:58.407 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-03-29T04:23:58.460 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-03-29T04:23:58.490 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:23:58.678 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install libcephfs2 2024-03-29T04:23:59.256 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:39 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:23:59.304 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:23:59.304 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/6 2024-03-29T04:23:59.304 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-03-29T04:23:59.304 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-03-29T04:23:59.304 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-03-29T04:23:59.304 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-03-29T04:23:59.359 INFO:teuthology.orchestra.run.smithi029.stdout:Package libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:23:59.419 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:23:59.420 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:23:59.420 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:23:59.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install libcephfs-devel 2024-03-29T04:23:59.601 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:23:59.602 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:23:59.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-fuse 2024-03-29T04:24:00.057 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:40 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:00.214 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:00.215 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:24:00.216 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:00.216 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 25 k 2024-03-29T04:24:00.216 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 110 k 2024-03-29T04:24:00.216 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:24:00.387 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:35 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:00.469 INFO:teuthology.orchestra.run.smithi029.stdout:libcephfs-devel-17.2.7-767.g9b4b8ffd.el9.x86_64 98 kB/s | 25 kB 00:00 2024-03-29T04:24:00.470 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:00.470 INFO:teuthology.orchestra.run.smithi029.stdout:Total 98 kB/s | 25 kB 00:00 2024-03-29T04:24:00.470 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:24:00.474 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:24:00.474 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:24:00.475 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:24:00.476 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:24:00.538 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 812 k 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-29T04:24:00.539 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:00.540 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:24:00.540 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:00.540 INFO:teuthology.orchestra.run.smithi099.stdout:Install 2 Packages 2024-03-29T04:24:00.540 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:00.540 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 891 k 2024-03-29T04:24:00.540 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 2.6 M 2024-03-29T04:24:00.541 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:24:00.557 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:24:00.661 INFO:teuthology.orchestra.run.smithi099.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 651 kB/s | 79 kB 00:00 2024-03-29T04:24:00.681 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:01.053 INFO:teuthology.orchestra.run.smithi099.stdout:(2/2): ceph-fuse-17.2.7-767.g9b4b8ffd.el9.x86_6 1.5 MB/s | 812 kB 00:00 2024-03-29T04:24:01.054 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:01.055 INFO:teuthology.orchestra.run.smithi099.stdout:Total 1.7 MB/s | 891 kB 00:00 2024-03-29T04:24:01.055 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:24:01.076 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:24:01.076 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:24:01.114 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:01.147 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:24:01.147 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:24:01.334 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:24:01.358 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:01.358 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:01.358 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:24:01.358 INFO:teuthology.orchestra.run.smithi029.stdout: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:01.359 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:01.359 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-03-29T04:24:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:24:01.571 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librados2 2024-03-29T04:24:02.143 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:42 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:02.234 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:24:02.234 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/2 2024-03-29T04:24:02.244 INFO:teuthology.orchestra.run.smithi029.stdout:Package librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:02.300 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:02.301 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:24:02.302 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:02.371 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install librbd1 2024-03-29T04:24:02.519 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-03-29T04:24:02.519 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:02.519 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:24:02.519 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-03-29T04:24:02.519 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:02.519 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:02.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install librados-devel 2024-03-29T04:24:02.943 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:42 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:03.043 INFO:teuthology.orchestra.run.smithi029.stdout:Package librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:03.099 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:03.100 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:24:03.100 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:03.180 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rados 2024-03-29T04:24:03.343 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:38 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:03.498 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:03.498 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout: librados-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 126 k 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 126 k 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 448 k 2024-03-29T04:24:03.499 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:24:03.754 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:43 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:03.816 INFO:teuthology.orchestra.run.smithi099.stdout:librados-devel-17.2.7-767.g9b4b8ffd.el9.x86_64. 399 kB/s | 126 kB 00:00 2024-03-29T04:24:03.816 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:03.817 INFO:teuthology.orchestra.run.smithi099.stdout:Total 397 kB/s | 126 kB 00:00 2024-03-29T04:24:03.817 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:24:03.822 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:24:03.822 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:24:03.856 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:03.877 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:24:03.877 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:24:03.911 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:03.913 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:24:03.913 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:03.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rgw 2024-03-29T04:24:04.031 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:24:04.139 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:04.555 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:44 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:04.558 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:04.657 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:04.713 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:04.714 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:24:04.714 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:04.785 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-cephfs 2024-03-29T04:24:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:04.820 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:24:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:04.820 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:04.820 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:05.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install libcephfs2 2024-03-29T04:24:05.357 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:45 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:05.457 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:05.512 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:05.514 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:24:05.514 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:05.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install python3-rbd 2024-03-29T04:24:05.626 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:40 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:05.726 INFO:teuthology.orchestra.run.smithi099.stdout:Package libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:05.780 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:05.781 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:05.781 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:05.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install libcephfs-devel 2024-03-29T04:24:06.156 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:46 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:06.258 INFO:teuthology.orchestra.run.smithi029.stdout:Package python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:06.313 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:06.314 INFO:teuthology.orchestra.run.smithi029.stdout:Nothing to do. 2024-03-29T04:24:06.314 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:06.398 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-fuse 2024-03-29T04:24:06.404 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:41 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:06.554 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:06.555 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 25 k 2024-03-29T04:24:06.556 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 110 k 2024-03-29T04:24:06.556 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:24:06.712 INFO:teuthology.orchestra.run.smithi099.stdout:libcephfs-devel-17.2.7-767.g9b4b8ffd.el9.x86_64 160 kB/s | 25 kB 00:00 2024-03-29T04:24:06.713 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:06.713 INFO:teuthology.orchestra.run.smithi099.stdout:Total 158 kB/s | 25 kB 00:00 2024-03-29T04:24:06.713 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:24:06.716 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:24:06.716 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:24:06.718 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:24:06.718 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:24:06.793 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:24:06.892 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:06.966 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:46 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:07.123 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:07.124 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:07.124 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2024-03-29T04:24:07.124 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:07.124 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:24:07.124 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 82 k 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 82 k 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 222 k 2024-03-29T04:24:07.125 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:24:07.262 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:07.327 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-fuse-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 407 kB/s | 82 kB 00:00 2024-03-29T04:24:07.329 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:07.329 INFO:teuthology.orchestra.run.smithi029.stdout:Total 404 kB/s | 82 kB 00:00 2024-03-29T04:24:07.329 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:24:07.335 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:24:07.335 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:24:07.392 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:24:07.392 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:24:07.517 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:07.517 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:07.518 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:24:07.518 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:07.518 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:07.518 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:07.558 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:24:07.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install librados2 2024-03-29T04:24:07.679 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:08.114 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:08.237 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:43 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:08.335 INFO:teuthology.orchestra.run.smithi099.stdout:Package librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:08.359 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:08.360 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:08.360 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:24:08.360 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:08.360 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:08.360 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:08.387 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:08.388 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:08.388 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:08.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install librbd1 2024-03-29T04:24:08.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-mirror 2024-03-29T04:24:09.012 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:44 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:09.110 INFO:teuthology.orchestra.run.smithi099.stdout:Package librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:09.162 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:09.163 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:09.163 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:09.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-rados 2024-03-29T04:24:09.246 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:49 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:09.407 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout: Package Arch Version Repo Size 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-mirror x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 2.9 M 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 12 M 2024-03-29T04:24:09.408 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:24:09.796 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:44 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:09.894 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:09.946 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:09.947 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:09.947 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:10.006 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-mirror-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 4.9 MB/s | 2.9 MB 00:00 2024-03-29T04:24:10.007 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:10.007 INFO:teuthology.orchestra.run.smithi029.stdout:Total 4.9 MB/s | 2.9 MB 00:00 2024-03-29T04:24:10.007 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:24:10.017 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:24:10.018 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:24:10.020 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-rgw 2024-03-29T04:24:10.095 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:24:10.095 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:24:10.520 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:24:10.577 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:45 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:10.591 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:10.616 INFO:teuthology.orchestra.run.smithi029.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:24:10.616 INFO:teuthology.orchestra.run.smithi029.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-29T04:24:10.616 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:24:10.616 INFO:teuthology.orchestra.run.smithi029.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:24:10.616 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:10.675 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:10.727 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:10.728 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:10.728 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:10.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-cephfs 2024-03-29T04:24:11.352 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:46 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:11.449 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:11.501 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:11.502 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:11.502 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:11.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-rbd 2024-03-29T04:24:11.897 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:11.898 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:24:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:11.898 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:11.898 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:12.128 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:47 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:12.187 DEBUG:teuthology.orchestra.run.smithi029:> sudo yum -y install rbd-nbd 2024-03-29T04:24:12.226 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:24:12.278 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:12.279 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2024-03-29T04:24:12.279 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:12.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install rbd-fuse 2024-03-29T04:24:12.764 INFO:teuthology.orchestra.run.smithi029.stdout:Last metadata expiration check: 0:01:52 ago on Fri 29 Mar 2024 04:22:20 AM UTC. 2024-03-29T04:24:12.899 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:47 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:12.920 INFO:teuthology.orchestra.run.smithi029.stdout:Dependencies resolved. 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout: Package Architecture Version Repository Size 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout:Installing: 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-nbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 165 k 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction Summary 2024-03-29T04:24:12.921 INFO:teuthology.orchestra.run.smithi029.stdout:================================================================================ 2024-03-29T04:24:12.922 INFO:teuthology.orchestra.run.smithi029.stdout:Install 1 Package 2024-03-29T04:24:12.922 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:12.922 INFO:teuthology.orchestra.run.smithi029.stdout:Total download size: 165 k 2024-03-29T04:24:12.922 INFO:teuthology.orchestra.run.smithi029.stdout:Installed size: 477 k 2024-03-29T04:24:12.922 INFO:teuthology.orchestra.run.smithi029.stdout:Downloading Packages: 2024-03-29T04:24:13.050 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:13.051 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:13.051 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2024-03-29T04:24:13.051 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:13.051 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 82 k 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 82 k 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 222 k 2024-03-29T04:24:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:24:13.283 INFO:teuthology.orchestra.run.smithi029.stdout:rbd-nbd-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 457 kB/s | 165 kB 00:00 2024-03-29T04:24:13.284 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:13.284 INFO:teuthology.orchestra.run.smithi029.stdout:Total 456 kB/s | 165 kB 00:00 2024-03-29T04:24:13.284 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction check 2024-03-29T04:24:13.293 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction check succeeded. 2024-03-29T04:24:13.293 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction test 2024-03-29T04:24:13.351 INFO:teuthology.orchestra.run.smithi029.stdout:Transaction test succeeded. 2024-03-29T04:24:13.352 INFO:teuthology.orchestra.run.smithi029.stdout:Running transaction 2024-03-29T04:24:13.356 INFO:teuthology.orchestra.run.smithi099.stdout:rbd-fuse-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 271 kB/s | 82 kB 00:00 2024-03-29T04:24:13.356 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:13.357 INFO:teuthology.orchestra.run.smithi099.stdout:Total 270 kB/s | 82 kB 00:00 2024-03-29T04:24:13.357 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:24:13.364 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:24:13.364 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:24:13.419 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:24:13.420 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:24:13.501 INFO:teuthology.orchestra.run.smithi029.stdout: Preparing : 1/1 2024-03-29T04:24:13.557 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:24:13.610 INFO:teuthology.orchestra.run.smithi029.stdout: Installing : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:13.671 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:14.055 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:14.088 INFO:teuthology.orchestra.run.smithi029.stdout: Running scriptlet: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:14.356 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:14.356 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:14.357 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:24:14.357 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:14.357 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:14.357 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: Verifying : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:14.404 INFO:teuthology.orchestra.run.smithi029.stdout:Installed: 2024-03-29T04:24:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:14.404 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:14.405 INFO:teuthology.orchestra.run.smithi029.stdout:Complete! 2024-03-29T04:24:14.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install rbd-mirror 2024-03-29T04:24:14.706 DEBUG:teuthology.parallel:result is None 2024-03-29T04:24:15.170 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:50 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:15.323 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:15.324 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:15.324 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2024-03-29T04:24:15.324 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:15.324 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:24:15.324 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-mirror x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:24:15.324 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 2.9 M 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 12 M 2024-03-29T04:24:15.325 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:24:15.892 INFO:teuthology.orchestra.run.smithi099.stdout:rbd-mirror-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 5.2 MB/s | 2.9 MB 00:00 2024-03-29T04:24:15.892 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:15.893 INFO:teuthology.orchestra.run.smithi099.stdout:Total 5.2 MB/s | 2.9 MB 00:00 2024-03-29T04:24:15.893 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:24:15.902 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:24:15.902 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:24:15.974 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:24:15.974 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:24:16.389 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:24:16.442 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:16.467 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:16.467 INFO:teuthology.orchestra.run.smithi099.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:24:16.467 INFO:teuthology.orchestra.run.smithi099.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-29T04:24:16.467 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:24:16.467 INFO:teuthology.orchestra.run.smithi099.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:24:16.467 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:17.540 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:17.540 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:17.540 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:24:17.540 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:17.540 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:17.540 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:17.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install rbd-nbd 2024-03-29T04:24:18.338 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:53 ago on Fri 29 Mar 2024 04:22:25 AM UTC. 2024-03-29T04:24:18.492 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-nbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 165 k 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2024-03-29T04:24:18.493 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:18.494 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 165 k 2024-03-29T04:24:18.494 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 477 k 2024-03-29T04:24:18.494 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2024-03-29T04:24:18.728 INFO:teuthology.orchestra.run.smithi099.stdout:rbd-nbd-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 708 kB/s | 165 kB 00:00 2024-03-29T04:24:18.729 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:18.729 INFO:teuthology.orchestra.run.smithi099.stdout:Total 702 kB/s | 165 kB 00:00 2024-03-29T04:24:18.729 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2024-03-29T04:24:18.738 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2024-03-29T04:24:18.738 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2024-03-29T04:24:18.794 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2024-03-29T04:24:18.795 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2024-03-29T04:24:18.950 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2024-03-29T04:24:19.060 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:19.541 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:19.806 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:24:19.806 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:19.806 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2024-03-29T04:24:19.806 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:24:19.806 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:24:19.806 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2024-03-29T04:24:20.031 DEBUG:teuthology.parallel:result is None 2024-03-29T04:24:20.031 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:24:20.032 INFO:teuthology.packaging:ref: None 2024-03-29T04:24:20.032 INFO:teuthology.packaging:tag: None 2024-03-29T04:24:20.032 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:24:20.032 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:24:20.032 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:24:20.244 DEBUG:teuthology.orchestra.run.smithi029:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-29T04:24:20.269 INFO:teuthology.orchestra.run.smithi029.stdout:17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:24:20.270 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:24:20.270 INFO:teuthology.task.install:The correct ceph version 17.2.7-767.g9b4b8ffd is installed. 2024-03-29T04:24:20.272 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:24:20.272 INFO:teuthology.packaging:ref: None 2024-03-29T04:24:20.272 INFO:teuthology.packaging:tag: None 2024-03-29T04:24:20.272 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:24:20.272 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:24:20.272 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:24:20.476 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-29T04:24:20.497 INFO:teuthology.orchestra.run.smithi099.stdout:17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:24:20.497 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:24:20.497 INFO:teuthology.task.install:The correct ceph version 17.2.7-767.g9b4b8ffd is installed. 2024-03-29T04:24:20.499 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-29T04:24:20.499 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:24:20.499 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-29T04:24:20.531 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:24:20.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-29T04:24:20.566 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-29T04:24:20.566 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:24:20.566 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/daemon-helper 2024-03-29T04:24:20.603 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-29T04:24:20.674 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:24:20.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/daemon-helper 2024-03-29T04:24:20.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-29T04:24:20.768 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-29T04:24:20.768 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:24:20.768 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-29T04:24:20.799 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-29T04:24:20.868 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:24:20.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-29T04:24:20.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-29T04:24:20.962 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-29T04:24:20.962 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:24:20.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/usr/bin/stdin-killer 2024-03-29T04:24:20.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-29T04:24:21.063 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:24:21.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/stdin-killer 2024-03-29T04:24:21.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-29T04:24:21.157 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-29T04:24:21.252 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'} 2024-03-29T04:24:21.252 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:24:21.252 INFO:tasks.cephadm:Cluster fsid is 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:24:21.253 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-29T04:24:21.253 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.29', 'mon.c': '[v2:172.21.15.29:3301,v1:172.21.15.29:6790]', 'mon.b': '172.21.15.99'} 2024-03-29T04:24:21.253 INFO:tasks.cephadm:First mon is mon.a on smithi029 2024-03-29T04:24:21.253 INFO:tasks.cephadm:First mgr is y 2024-03-29T04:24:21.253 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-29T04:24:21.253 DEBUG:teuthology.orchestra.run.smithi029:> sudo hostname $(hostname -s) 2024-03-29T04:24:21.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo hostname $(hostname -s) 2024-03-29T04:24:21.313 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-03-29T04:24:21.313 DEBUG:teuthology.orchestra.run.smithi029:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-29T04:24:21.562 INFO:teuthology.orchestra.run.smithi029.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Mar 29 04:24 /home/ubuntu/cephtest/cephadm 2024-03-29T04:24:21.562 DEBUG:teuthology.orchestra.run.smithi099:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-29T04:24:21.757 INFO:teuthology.orchestra.run.smithi099.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Mar 29 04:24 /home/ubuntu/cephtest/cephadm 2024-03-29T04:24:21.758 DEBUG:teuthology.orchestra.run.smithi029:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-29T04:24:21.779 DEBUG:teuthology.orchestra.run.smithi099:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-29T04:24:21.806 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-03-29T04:24:21.806 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-03-29T04:24:21.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-03-29T04:24:22.747 INFO:teuthology.orchestra.run.smithi099.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:24:22.840 INFO:teuthology.orchestra.run.smithi029.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:24:48.075 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-03-29T04:24:48.075 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph_version": "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)", 2024-03-29T04:24:48.076 INFO:teuthology.orchestra.run.smithi029.stdout: "image_id": "cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a", 2024-03-29T04:24:48.076 INFO:teuthology.orchestra.run.smithi029.stdout: "repo_digests": [ 2024-03-29T04:24:48.076 INFO:teuthology.orchestra.run.smithi029.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b" 2024-03-29T04:24:48.076 INFO:teuthology.orchestra.run.smithi029.stdout: ] 2024-03-29T04:24:48.076 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-03-29T04:24:51.568 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2024-03-29T04:24:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_version": "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)", 2024-03-29T04:24:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "image_id": "cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a", 2024-03-29T04:24:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "repo_digests": [ 2024-03-29T04:24:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b" 2024-03-29T04:24:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2024-03-29T04:24:51.569 INFO:teuthology.orchestra.run.smithi099.stdout:} 2024-03-29T04:24:51.588 DEBUG:teuthology.orchestra.run.smithi029:> sudo mkdir -p /etc/ceph 2024-03-29T04:24:51.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo mkdir -p /etc/ceph 2024-03-29T04:24:51.653 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 777 /etc/ceph 2024-03-29T04:24:51.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 777 /etc/ceph 2024-03-29T04:24:51.720 INFO:tasks.cephadm:Writing seed config... 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-29T04:24:51.721 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-29T04:24:51.722 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-29T04:24:51.722 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-29T04:24:51.722 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-29T04:24:51.722 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:24:51.722 DEBUG:teuthology.orchestra.run.smithi029:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-29T04:24:51.751 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 37319ec8-ed84-11ee-95d0-87774f69a715 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-29T04:24:51.751 DEBUG:teuthology.orchestra.run.smithi029:mon.a> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service 2024-03-29T04:24:51.794 DEBUG:teuthology.orchestra.run.smithi029:mgr.y> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service 2024-03-29T04:24:51.836 INFO:tasks.cephadm:Bootstrapping... 2024-03-29T04:24:51.836 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.29 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:24:52.051 INFO:teuthology.orchestra.run.smithi029.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:24:52.051 INFO:teuthology.orchestra.run.smithi029.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '37319ec8-ed84-11ee-95d0-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.29', '--skip-admin-label'] 2024-03-29T04:24:52.077 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:24:52.077 INFO:teuthology.orchestra.run.smithi029.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-03-29T04:24:52.078 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying podman|docker is present... 2024-03-29T04:24:52.101 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:24:52.101 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying lvm2 is present... 2024-03-29T04:24:52.101 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying time synchronization is in place... 2024-03-29T04:24:52.108 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-29T04:24:52.108 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-29T04:24:52.114 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-29T04:24:52.114 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-03-29T04:24:52.121 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout enabled 2024-03-29T04:24:52.127 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout active 2024-03-29T04:24:52.127 INFO:teuthology.orchestra.run.smithi029.stdout:Unit chronyd.service is enabled and running 2024-03-29T04:24:52.127 INFO:teuthology.orchestra.run.smithi029.stdout:Repeating the final host check... 2024-03-29T04:24:52.151 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:24:52.151 INFO:teuthology.orchestra.run.smithi029.stdout:podman (/bin/podman) version 4.9.4 is present 2024-03-29T04:24:52.151 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl is present 2024-03-29T04:24:52.151 INFO:teuthology.orchestra.run.smithi029.stdout:lvcreate is present 2024-03-29T04:24:52.157 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-29T04:24:52.157 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-29T04:24:52.163 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-29T04:24:52.163 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-03-29T04:24:52.170 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout enabled 2024-03-29T04:24:52.176 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout active 2024-03-29T04:24:52.176 INFO:teuthology.orchestra.run.smithi029.stdout:Unit chronyd.service is enabled and running 2024-03-29T04:24:52.176 INFO:teuthology.orchestra.run.smithi029.stdout:Host looks OK 2024-03-29T04:24:52.176 INFO:teuthology.orchestra.run.smithi029.stdout:Cluster fsid: 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:24:52.177 INFO:teuthology.orchestra.run.smithi029.stdout:Acquiring lock 140107847337056 on /run/cephadm/37319ec8-ed84-11ee-95d0-87774f69a715.lock 2024-03-29T04:24:52.177 INFO:teuthology.orchestra.run.smithi029.stdout:Lock 140107847337056 acquired on /run/cephadm/37319ec8-ed84-11ee-95d0-87774f69a715.lock 2024-03-29T04:24:52.177 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying IP 172.21.15.29 port 3300 ... 2024-03-29T04:24:52.177 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying IP 172.21.15.29 port 6789 ... 2024-03-29T04:24:52.177 INFO:teuthology.orchestra.run.smithi029.stdout:Base mon IP(s) is [172.21.15.29:3300, 172.21.15.29:6789], mon addrv is [v2:172.21.15.29:3300,v1:172.21.15.29:6789] 2024-03-29T04:24:52.181 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.29 metric 100 2024-03-29T04:24:52.181 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.29 metric 100 2024-03-29T04:24:52.184 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-03-29T04:24:52.184 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-03-29T04:24:52.184 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-03-29T04:24:52.186 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-29T04:24:52.186 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-03-29T04:24:52.186 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-29T04:24:52.186 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-03-29T04:24:52.187 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:125a/64 scope link noprefixroute 2024-03-29T04:24:52.187 INFO:teuthology.orchestra.run.smithi029.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-29T04:24:52.187 INFO:teuthology.orchestra.run.smithi029.stdout:Mon IP `172.21.15.29` is in CIDR network `172.21.0.0/20` 2024-03-29T04:24:52.187 INFO:teuthology.orchestra.run.smithi029.stdout:Mon IP `172.21.15.29` is in CIDR network `172.21.0.0/20` 2024-03-29T04:24:52.187 INFO:teuthology.orchestra.run.smithi029.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-03-29T04:24:52.188 INFO:teuthology.orchestra.run.smithi029.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-29T04:24:52.188 INFO:teuthology.orchestra.run.smithi029.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:24:52.686 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stdout cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a 2024-03-29T04:24:52.686 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:24:52.686 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Getting image source signatures 2024-03-29T04:24:52.687 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying blob sha256:11da15500eca1f826c328eb3265caa30b0a3730510e2a32023084d6fe9cee20c 2024-03-29T04:24:52.687 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying blob sha256:7ae200e79e73f55e16b47791a08422a21d8b9f8d5bb767e6ae856e160a74c37a 2024-03-29T04:24:52.687 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Copying config sha256:cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a 2024-03-29T04:24:52.687 INFO:teuthology.orchestra.run.smithi029.stdout:/bin/podman: stderr Writing manifest to image destination 2024-03-29T04:24:53.699 INFO:teuthology.orchestra.run.smithi029.stdout:ceph: stdout ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable) 2024-03-29T04:24:53.699 INFO:teuthology.orchestra.run.smithi029.stdout:Ceph version: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable) 2024-03-29T04:24:53.699 INFO:teuthology.orchestra.run.smithi029.stdout:Extracting ceph user uid/gid from container image... 2024-03-29T04:24:54.745 INFO:teuthology.orchestra.run.smithi029.stdout:stat: stdout 167 167 2024-03-29T04:24:54.745 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial keys... 2024-03-29T04:24:55.683 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQCXQgZmfcjGBxAAxqKBWPuei0W06Z03oYrU8w== 2024-03-29T04:24:56.653 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQCYQgZm5BsBBBAAnyjelKKOpNVIJFOI1MvPew== 2024-03-29T04:24:57.649 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph-authtool: stdout AQCZQgZm+6nhAhAA0DYblYoLKuHrEnQtTsUeVw== 2024-03-29T04:24:57.650 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial monmap... 2024-03-29T04:24:58.908 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:monmaptool for a [v2:172.21.15.29:3300,v1:172.21.15.29:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:setting min_mon_release = octopus 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: set fsid to 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:24:58.909 INFO:teuthology.orchestra.run.smithi029.stdout:Creating mon... 2024-03-29T04:24:59.887 INFO:teuthology.orchestra.run.smithi029.stdout:create mon.a on 2024-03-29T04:25:00.096 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-03-29T04:25:00.235 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-29T04:25:00.384 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-37319ec8-ed84-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-37319ec8-ed84-11ee-95d0-87774f69a715.target. 2024-03-29T04:25:00.384 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-37319ec8-ed84-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-37319ec8-ed84-11ee-95d0-87774f69a715.target. 2024-03-29T04:25:00.551 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a 2024-03-29T04:25:00.552 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to reset failed state of unit ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service: Unit ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service not loaded. 2024-03-29T04:25:00.714 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-37319ec8-ed84-11ee-95d0-87774f69a715.target.wants/ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service → /etc/systemd/system/ceph-37319ec8-ed84-11ee-95d0-87774f69a715@.service. 2024-03-29T04:25:01.235 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:01 smithi029 ceph-mon[29557]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:25:01.235 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:01 smithi029 bash[29519]: 7aed5820abe2243bab4a6abd4e6414ac1cb42aef35aec4ad05717e4c053a9ee6 2024-03-29T04:25:01.248 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:25:01.248 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-03-29T04:25:01.255 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:25:01.256 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-03-29T04:25:01.256 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-03-29T04:25:01.256 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to enable service . firewalld.service is not available 2024-03-29T04:25:01.256 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mon to start... 2024-03-29T04:25:01.256 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mon... 2024-03-29T04:25:01.494 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:01 smithi029 systemd[1]: Started Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:25:02.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:02 smithi029 ceph-mon[29557]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:25:02.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:02 smithi029 ceph-mon[29557]: monmap e1: 1 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:25:02.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:02 smithi029 ceph-mon[29557]: fsmap 2024-03-29T04:25:02.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:02 smithi029 ceph-mon[29557]: osdmap e1: 0 total, 0 up, 0 in 2024-03-29T04:25:02.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:02 smithi029 ceph-mon[29557]: mgrmap e1: no daemons active 2024-03-29T04:25:02.342 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:02 smithi029 ceph-mon[29557]: from='client.? 172.21.15.29:0/148564464' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-29T04:25:02.348 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout cluster: 2024-03-29T04:25:02.348 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout id: 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:02.348 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-03-29T04:25:02.348 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:02.348 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout services: 2024-03-29T04:25:02.348 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.613645s) 2024-03-29T04:25:02.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-03-29T04:25:02.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-03-29T04:25:02.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:02.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout data: 2024-03-29T04:25:02.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-03-29T04:25:02.349 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-03-29T04:25:02.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-03-29T04:25:02.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout pgs: 2024-03-29T04:25:02.350 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:02.350 INFO:teuthology.orchestra.run.smithi029.stdout:mon is available 2024-03-29T04:25:02.350 INFO:teuthology.orchestra.run.smithi029.stdout:Assimilating anything we can from ceph.conf... 2024-03-29T04:25:03.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:03 smithi029 ceph-mon[29557]: from='client.? 172.21.15.29:0/1129752471' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-29T04:25:03.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:03 smithi029 ceph-mon[29557]: from='client.? 172.21.15.29:0/1129752471' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-29T04:25:03.443 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:03.443 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [global] 2024-03-29T04:25:03.443 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout fsid = 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:03.443 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.29:3300,v1:172.21.15.29:6789] 2024-03-29T04:25:03.443 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-29T04:25:03.443 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [osd] 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-29T04:25:03.444 INFO:teuthology.orchestra.run.smithi029.stdout:Generating new minimal ceph.conf... 2024-03-29T04:25:04.634 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:04 smithi029 ceph-mon[29557]: from='client.? 172.21.15.29:0/2105421140' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:25:08.552 INFO:teuthology.orchestra.run.smithi029.stdout:Restarting the monitor... 2024-03-29T04:25:08.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:08 smithi029 systemd[1]: Stopping Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:25:10.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[29534]: 2024-03-29T04:25:10.097+0000 7f176ac25700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:25:10.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[29534]: 2024-03-29T04:25:10.097+0000 7f176ac25700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-03-29T04:25:10.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:10 smithi029 podman[29879]: 2024-03-29 04:25:10.303765559 +0000 UTC m=+1.558707237 container died 7aed5820abe2243bab4a6abd4e6414ac1cb42aef35aec4ad05717e4c053a9ee6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, io.buildah.version=1.33.5) 2024-03-29T04:25:16.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:16 smithi029 podman[29879]: 2024-03-29 04:25:16.341587763 +0000 UTC m=+7.596529432 container cleanup 7aed5820abe2243bab4a6abd4e6414ac1cb42aef35aec4ad05717e4c053a9ee6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:25:16.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:16 smithi029 bash[29879]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:25:19.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:19 smithi029 podman[29899]: 2024-03-29 04:25:19.355544884 +0000 UTC m=+9.041081197 container remove 7aed5820abe2243bab4a6abd4e6414ac1cb42aef35aec4ad05717e4c053a9ee6 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-29T04:25:19.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:19 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T04:25:19.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:19 smithi029 systemd[1]: Stopped Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:25:19.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:19 smithi029 systemd[1]: Starting Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:25:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:19 smithi029 podman[29996]: 2024-03-29 04:25:19.711528046 +0000 UTC m=+0.018116296 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:25:21.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:20 smithi029 podman[29996]: 2024-03-29 04:25:20.976213169 +0000 UTC m=+1.282801391 container create 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 podman[29996]: 2024-03-29 04:25:24.437684951 +0000 UTC m=+4.744273187 container init 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 podman[29996]: 2024-03-29 04:25:24.44443406 +0000 UTC m=+4.751022284 container start 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, ceph=True, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable), process ceph-mon, pid 7 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: pidfile_write: ignore empty --pid-file 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: load: jerasure load: lrc 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: RocksDB version: 6.15.5 2024-03-29T04:25:24.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Compile date Mar 28 2024 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: DB SUMMARY 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: DB Session ID: V9ZGLSNGHBYH23Y4J26N 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84215 ; 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:25:24.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.env: 0x559217147a80 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.fs: Posix File System 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.info_log: 0x559218650e80 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.statistics: (nil) 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.use_fsync: 0 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:25:24.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.db_log_dir: 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.write_buffer_manager: 0x5592193c0960 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:25:24.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.unordered_write: 0 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.row_cache: None 2024-03-29T04:25:24.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.wal_filter: None 2024-03-29T04:25:24.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:25:24.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:25:24.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.preserve_deletes: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:25:24.676 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:25:24.677 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_open_files: -1 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Compression algorithms supported: 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kZSTD supported: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:25:24.678 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.merge_operator: 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_filter: None 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5592186a0f58) 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:25:24.679 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_type: 0 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_index_type: 0 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_shortening: 1 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: hash_index_allow_collision: 1 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: checksum: 1 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: no_block_cache: 0 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache: 0x5592186d3090 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_options: 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: capacity : 536870912 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_shard_bits : 4 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: strict_capacity_limit : 0 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:25:24.680 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_compressed: (nil) 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: persistent_cache: (nil) 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size: 4096 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size_deviation: 10 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_restart_interval: 16 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_block_restart_interval: 1 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_block_size: 4096 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: partition_filters: 0 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: use_delta_encoding: 1 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: whole_key_filtering: 1 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: verify_compression: 0 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: format_version: 4 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_index_compression: 1 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_align: 0 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:25:24.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression: NoCompression 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.num_levels: 7 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:25:24.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:25:24.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.arena_block_size: 4194304 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:25:24.684 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:25:24.685 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.ttl: 2592000 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-03-29T04:25:24.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686324475662, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686324476481, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80647, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78908, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73117, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711686324, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "fdb118fa-1198-411e-8d34-5ee0f70cc4e1", "db_session_id": "V9ZGLSNGHBYH23Y4J26N"}} 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686324476908, "job": 1, "event": "recovery_finished"} 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x559218667180 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: DB pointer 0x55921873c000 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** DB Stats ** 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:25:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 2/0 80.52 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 80.52 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.688 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 21.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 21.76 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:25:24.689 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 2/0 80.52 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 80.52 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.6 0.00 0.00 1 0.001 0 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 21.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:25:24.690 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: starting mon.a rank 0 at public addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] at bind addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???) e1 preinit fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).mds e1 new map 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).mds e1 print_map 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: e1 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: legacy client fscid: -1 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout: No filesystems configured 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:25:24.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: monmap e1: 1 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: fsmap 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: osdmap e1: 0 total, 0 up, 0 in 2024-03-29T04:25:24.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:24 smithi029 ceph-mon[30044]: mgrmap e1: no daemons active 2024-03-29T04:25:25.881 INFO:teuthology.orchestra.run.smithi029.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-03-29T04:25:26.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:25 smithi029 bash[29996]: 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 2024-03-29T04:25:26.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:25 smithi029 systemd[1]: Started Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:25:28.763 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote config to /etc/ceph/ceph.conf 2024-03-29T04:25:28.763 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:25:28.763 INFO:teuthology.orchestra.run.smithi029.stdout:Creating mgr... 2024-03-29T04:25:28.764 INFO:teuthology.orchestra.run.smithi029.stdout:Verifying port 9283 ... 2024-03-29T04:25:28.942 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y 2024-03-29T04:25:28.942 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Failed to reset failed state of unit ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service: Unit ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service not loaded. 2024-03-29T04:25:29.092 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-37319ec8-ed84-11ee-95d0-87774f69a715.target.wants/ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service → /etc/systemd/system/ceph-37319ec8-ed84-11ee-95d0-87774f69a715@.service. 2024-03-29T04:25:29.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:29 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/560520241' entity='client.admin' 2024-03-29T04:25:29.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:29 smithi029 podman[30288]: 2024-03-29 04:25:29.361790279 +0000 UTC m=+0.128396587 container create a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, ceph=True) 2024-03-29T04:25:29.674 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:25:29.674 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-03-29T04:25:29.681 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:25:29.681 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-03-29T04:25:29.681 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-03-29T04:25:29.681 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to enable service . firewalld.service is not available 2024-03-29T04:25:29.690 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:25:29.690 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-03-29T04:25:29.697 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:25:29.697 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-03-29T04:25:29.697 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-03-29T04:25:29.697 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-29T04:25:29.697 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr to start... 2024-03-29T04:25:29.698 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr... 2024-03-29T04:25:29.823 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:29 smithi029 podman[30288]: 2024-03-29 04:25:29.583870379 +0000 UTC m=+0.350476633 container init a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, GIT_CLEAN=True) 2024-03-29T04:25:29.823 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:29 smithi029 podman[30288]: 2024-03-29 04:25:29.589464433 +0000 UTC m=+0.356070689 container start a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-29T04:25:29.823 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:29 smithi029 bash[30288]: a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 2024-03-29T04:25:29.824 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:29 smithi029 systemd[1]: Started Ceph mgr.y for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:25:30.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:30 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1278265188' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:25:30.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:30.127+0000 7fdc947101c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:25:30.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:30.223+0000 7fdc947101c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:25:30.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:30.882+0000 7fdc947101c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:25:30.990 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:30.990 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:25:30.990 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "37319ec8-ed84-11ee-95d0-87774f69a715", 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-03-29T04:25:30.991 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:25:30.992 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:25:30.996 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:25:01.200243+0000", 2024-03-29T04:25:30.997 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:30.998 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:30.998 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:25:30.998 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:25:30.998 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (1/15)... 2024-03-29T04:25:31.267 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:30.973+0000 7fdc947101c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:25:31.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.059+0000 7fdc947101c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:25:31.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.209+0000 7fdc947101c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:25:31.268 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.291+0000 7fdc947101c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:25:31.624 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.463+0000 7fdc947101c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:25:31.624 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.647+0000 7fdc947101c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:25:31.889 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.732+0000 7fdc947101c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:25:31.889 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.825+0000 7fdc947101c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:25:31.889 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:31.912+0000 7fdc947101c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:25:33.225 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:32.990+0000 7fdc947101c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:25:33.226 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:33.129+0000 7fdc947101c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:25:33.535 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:33.326+0000 7fdc947101c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:25:33.535 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:33.558+0000 7fdc947101c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:25:33.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:33 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/180127256' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:25:33.897 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:33.738+0000 7fdc947101c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:25:34.160 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:33.921+0000 7fdc947101c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:25:34.197 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:34.197 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:25:34.197 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "37319ec8-ed84-11ee-95d0-87774f69a715", 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-03-29T04:25:34.198 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:25:34.199 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:25:34.200 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:25:34.200 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:25:34.202 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:25:34.202 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:25:34.202 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:25:34.202 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:25:34.202 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.203 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:25:01.200243+0000", 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:25:34.204 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (2/15)... 2024-03-29T04:25:34.418 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:34.183+0000 7fdc947101c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:25:34.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:34.269+0000 7fdc947101c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:25:34.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:34.661+0000 7fdc947101c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:25:35.729 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:35.455+0000 7fdc947101c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:35.752+0000 7fdc947101c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:36.067+0000 7fdc947101c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:36.264+0000 7fdc947101c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: Activating manager daemon y 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: mgrmap e2: y(active, starting, since 0.00262312s) 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:25:36.321 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:25:36.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: Manager daemon y is now available 2024-03-29T04:25:36.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:25:36.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:25:36.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' 2024-03-29T04:25:36.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' 2024-03-29T04:25:36.322 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:36 smithi029 ceph-mon[30044]: from='mgr.14100 172.21.15.29:0/2851331004' entity='mgr.y' 2024-03-29T04:25:37.348 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:37 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1290563458' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:25:37.348 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:37 smithi029 ceph-mon[30044]: mgrmap e3: y(active, since 1.00579s) 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "37319ec8-ed84-11ee-95d0-87774f69a715", 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:25:37.354 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:25:37.355 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:25:37.356 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:25:37.359 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:25:37.360 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:25:01.200243+0000", 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:25:37.361 INFO:teuthology.orchestra.run.smithi029.stdout:mgr not available, waiting (3/15)... 2024-03-29T04:25:39.608 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:39 smithi029 ceph-mon[30044]: mgrmap e4: y(active, since 2s) 2024-03-29T04:25:40.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:40 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1502289181' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:25:40.842 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:40.842 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsid": "37319ec8-ed84-11ee-95d0-87774f69a715", 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:40.843 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "quorum_age": 15, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:25:40.844 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.848 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:25:01.200243+0000", 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:25:40.850 INFO:teuthology.orchestra.run.smithi029.stdout:mgr is available 2024-03-29T04:25:41.865 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2774535226' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-29T04:25:42.156 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [global] 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout fsid = 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout [osd] 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-29T04:25:42.157 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling cephadm module... 2024-03-29T04:25:43.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:42 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4093915314' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-29T04:25:43.239 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:43 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setuser ceph since I am not root 2024-03-29T04:25:43.240 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:43 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setgroup ceph since I am not root 2024-03-29T04:25:43.516 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:43 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:43.497+0000 7f985d37f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:25:43.769 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:43 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:43.594+0000 7f985d37f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:25:44.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:43 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4093915314' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-29T04:25:44.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:43 smithi029 ceph-mon[30044]: mgrmap e5: y(active, since 6s) 2024-03-29T04:25:44.554 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:44.247+0000 7f985d37f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:25:44.554 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:44.341+0000 7f985d37f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:25:44.555 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:44.426+0000 7f985d37f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:25:44.555 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:44.578+0000 7f985d37f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:25:44.811 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:44.661+0000 7f985d37f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:25:44.811 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:44.835+0000 7f985d37f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:25:44.851 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:25:44.851 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-03-29T04:25:44.851 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:25:44.851 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-29T04:25:44.852 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-29T04:25:44.852 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:25:44.852 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for the mgr to restart... 2024-03-29T04:25:44.852 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr epoch 5... 2024-03-29T04:25:45.087 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:44 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3491992540' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-29T04:25:45.088 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:45.023+0000 7f985d37f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:25:45.088 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:45.111+0000 7f985d37f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:25:45.418 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:45.208+0000 7f985d37f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:25:45.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:45.300+0000 7f985d37f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:25:46.662 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:46.360+0000 7f985d37f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:25:46.662 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:46.491+0000 7f985d37f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:25:46.663 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:46.686+0000 7f985d37f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:25:46.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:46.913+0000 7f985d37f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:25:47.418 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:47 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:47.089+0000 7f985d37f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:25:47.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:47 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:47.271+0000 7f985d37f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:25:47.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:47 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:47.532+0000 7f985d37f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:25:47.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:47 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:47.618+0000 7f985d37f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:25:48.418 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:48 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:48.009+0000 7f985d37f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:25:49.073 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:48 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:48.800+0000 7f985d37f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:25:49.074 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:49.097+0000 7f985d37f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:25:49.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:49.410+0000 7f985d37f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:25:49.670 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:25:49.605+0000 7f985d37f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:25:49.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: Active manager daemon y restarted 2024-03-29T04:25:49.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: Activating manager daemon y 2024-03-29T04:25:49.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: osdmap e2: 0 total, 0 up, 0 in 2024-03-29T04:25:49.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: mgrmap e6: y(active, starting, since 0.00301582s) 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: Manager daemon y is now available 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:25:49.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:25:49.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:25:49.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:49 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:25:50.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:50 smithi029 ceph-mon[30044]: Found migration_current of "None". Setting to last migration. 2024-03-29T04:25:50.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:50 smithi029 ceph-mon[30044]: mgrmap e7: y(active, since 1.00528s) 2024-03-29T04:25:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:25:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-03-29T04:25:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-29T04:25:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:25:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:mgr epoch 5 is available 2024-03-29T04:25:51.183 INFO:teuthology.orchestra.run.smithi029.stdout:Setting orchestrator backend to cephadm... 2024-03-29T04:25:51.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:51 smithi029 ceph-mon[30044]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-29T04:25:51.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:51 smithi029 ceph-mon[30044]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-29T04:25:51.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:51 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:25:51.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:51 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:25:53.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: [29/Mar/2024:04:25:51] ENGINE Bus STARTING 2024-03-29T04:25:53.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: [29/Mar/2024:04:25:51] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:25:53.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: [29/Mar/2024:04:25:51] ENGINE Bus STARTED 2024-03-29T04:25:53.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:25:53.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:25:53.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:25:53.163 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:52 smithi029 ceph-mon[30044]: mgrmap e8: y(active, since 2s) 2024-03-29T04:25:53.719 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout value unchanged 2024-03-29T04:25:53.720 INFO:teuthology.orchestra.run.smithi029.stdout:Generating ssh key... 2024-03-29T04:25:54.114 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:54 smithi029 ceph-mon[30044]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:25:55.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-mon[30044]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:25:55.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-mon[30044]: Generating ssh key... 2024-03-29T04:25:55.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: Generating public/private rsa key pair. 2024-03-29T04:25:55.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: Your identification has been saved in /tmp/tmp_povp2k2/key. 2024-03-29T04:25:55.419 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: Your public key has been saved in /tmp/tmp_povp2k2/key.pub. 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: The key fingerprint is: 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: SHA256:4hGeUA2DkGak0fQVI5xa/HO66qKtA2Cu9yx615XEiHQ ceph-37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: The key's randomart image is: 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: +---[RSA 3072]----+ 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |.++=.oB+ | 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: | o=.B+Eo. | 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |.o +o+.o | 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |... .o+o+ | 2024-03-29T04:25:55.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |+ ==S. | 2024-03-29T04:25:55.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |.. ..oo | 2024-03-29T04:25:55.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |o ..o | 2024-03-29T04:25:55.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |o.=.. o | 2024-03-29T04:25:55.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: |+B.*=. | 2024-03-29T04:25:55.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:25:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: +----[SHA256]-----+ 2024-03-29T04:25:56.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:56 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:25:56.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:56 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:25:57.024 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgC9dpq1PBP+ZLEEsYPg6unae/5cu1dVNrHOZL4A0p32vICXJOazQsotl9O3SczfE3jhSZ7j+3a2Oz81ZtTe92YUX2D0OsTstx8qt5wWjzj4ojbQNGroSF8sf13HNzugvnEIkzCeZsW8tetldqN2s473IeHHDzVTeqsRuFtoHho5+d1EC3/jeuax4rr1l1a2LU+3H7c4f7x40kd+Ja4SsRkl+JdyywgXl3yKaSuBkzfrhosrgCppPTDXaS1DSOMl7e8be+88CjaaS5ohzRVhp/gds4kBLso7L8oGrvg3ZVDIviuoB8O77AKSfslEOYhtdImH9dxDENAw/E3H+3y8Ly2FXsZzPtK/RTAkl28iCw4qXzU8hVGIu47tqLLQNTGyOA8PGgn1MaE8rxO3o06zl4Ipc7k7lq/N2X78nur3cqIUFWRVYoLtOlXYcBFUzfog6pp2HQPESkE/wVC2m7nanIWiDHE+Gmm67i6hZg4gL9TnyUtDnCsCaxmhIFPPvfJs8= ceph-37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:25:57.024 INFO:teuthology.orchestra.run.smithi029.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:25:57.024 INFO:teuthology.orchestra.run.smithi029.stdout:Adding key to root@localhost authorized_keys... 2024-03-29T04:25:57.025 INFO:teuthology.orchestra.run.smithi029.stdout:Adding host smithi029... 2024-03-29T04:25:57.273 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:57 smithi029 ceph-mon[30044]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:25:58.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:58 smithi029 ceph-mon[30044]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi029", "addr": "172.21.15.29", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:25:59.472 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:25:59 smithi029 ceph-mon[30044]: Deploying cephadm binary to smithi029 2024-03-29T04:26:00.452 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Added host 'smithi029' with addr '172.21.15.29' 2024-03-29T04:26:00.452 INFO:teuthology.orchestra.run.smithi029.stdout:Deploying unmanaged mon service... 2024-03-29T04:26:01.142 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:00 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:01.142 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:00 smithi029 ceph-mon[30044]: Added host smithi029 2024-03-29T04:26:01.142 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:00 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:01.814 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-03-29T04:26:01.814 INFO:teuthology.orchestra.run.smithi029.stdout:Deploying unmanaged mgr service... 2024-03-29T04:26:02.371 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:02 smithi029 ceph-mon[30044]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:02.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:02 smithi029 ceph-mon[30044]: Saving service mon spec with placement count:5 2024-03-29T04:26:02.372 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:02 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:03.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:03 smithi029 ceph-mon[30044]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:03.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:03 smithi029 ceph-mon[30044]: Saving service mgr spec with placement count:2 2024-03-29T04:26:03.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:03 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:03.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:03 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:03.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:03 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:03.900 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-03-29T04:26:05.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1459818244' entity='client.admin' 2024-03-29T04:26:06.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:06 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:06.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:06 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3315084896' entity='client.admin' 2024-03-29T04:26:06.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:06 smithi029 ceph-mon[30044]: from='mgr.14120 172.21.15.29:0/771281474' entity='mgr.y' 2024-03-29T04:26:07.180 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling the dashboard module... 2024-03-29T04:26:08.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:07 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3163931678' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-29T04:26:08.169 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setuser ceph since I am not root 2024-03-29T04:26:08.169 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setgroup ceph since I am not root 2024-03-29T04:26:08.898 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:08.530+0000 7f0ef554e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:26:08.898 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:08.626+0000 7f0ef554e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:26:09.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:08 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3163931678' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-29T04:26:09.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:08 smithi029 ceph-mon[30044]: mgrmap e9: y(active, since 18s) 2024-03-29T04:26:09.584 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:09.273+0000 7f0ef554e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:26:09.584 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:09.363+0000 7f0ef554e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:26:09.584 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:09.452+0000 7f0ef554e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:26:09.584 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:09.606+0000 7f0ef554e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:26:09.847 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:09.692+0000 7f0ef554e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:26:10.100 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:09.869+0000 7f0ef554e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:26:10.101 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:10.053+0000 7f0ef554e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:26:10.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:10 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2416159366' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-29T04:26:10.417 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:10.143+0000 7f0ef554e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:26:10.417 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:10.239+0000 7f0ef554e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:26:10.417 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:10.330+0000 7f0ef554e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:26:10.689 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:26:10.689 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-03-29T04:26:10.689 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:26:10.689 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-29T04:26:10.690 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-29T04:26:10.690 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:26:10.690 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for the mgr to restart... 2024-03-29T04:26:10.690 INFO:teuthology.orchestra.run.smithi029.stdout:Waiting for mgr epoch 9... 2024-03-29T04:26:11.669 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:11.413+0000 7f0ef554e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:26:11.669 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:11.545+0000 7f0ef554e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:26:12.127 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:11.742+0000 7f0ef554e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:26:12.128 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:11.971+0000 7f0ef554e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:26:12.128 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:12.150+0000 7f0ef554e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:26:12.418 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:12.333+0000 7f0ef554e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:26:12.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:12.598+0000 7f0ef554e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:26:12.919 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:12.686+0000 7f0ef554e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:26:13.418 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:13.082+0000 7f0ef554e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:26:14.161 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:13.881+0000 7f0ef554e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:26:14.161 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:14.184+0000 7f0ef554e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:26:14.730 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:14.500+0000 7f0ef554e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:26:14.731 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:26:14.700+0000 7f0ef554e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:26:15.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: Active manager daemon y restarted 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: Activating manager daemon y 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: osdmap e3: 0 total, 0 up, 0 in 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: mgrmap e10: y(active, starting, since 0.00308036s) 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: Manager daemon y is now available 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:15.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:15.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:26:15.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:26:16.530 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:26:16.530 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-03-29T04:26:16.530 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-29T04:26:16.530 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:26:16.531 INFO:teuthology.orchestra.run.smithi029.stdout:mgr epoch 9 is available 2024-03-29T04:26:16.531 INFO:teuthology.orchestra.run.smithi029.stdout:Generating a dashboard self-signed certificate... 2024-03-29T04:26:16.797 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:16 smithi029 ceph-mon[30044]: mgrmap e11: y(active, since 1.0053s) 2024-03-29T04:26:16.797 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:16 smithi029 ceph-mon[30044]: [29/Mar/2024:04:26:15] ENGINE Bus STARTING 2024-03-29T04:26:16.797 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:16 smithi029 ceph-mon[30044]: [29/Mar/2024:04:26:15] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:26:16.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:16 smithi029 ceph-mon[30044]: [29/Mar/2024:04:26:15] ENGINE Bus STARTED 2024-03-29T04:26:16.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:16 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:16.798 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:16 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:18.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:18.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:18 smithi029 ceph-mon[30044]: mgrmap e12: y(active, since 2s) 2024-03-29T04:26:18.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:18 smithi029 ceph-mon[30044]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:18.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:18.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:19.142 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-03-29T04:26:19.142 INFO:teuthology.orchestra.run.smithi029.stdout:Creating initial admin user... 2024-03-29T04:26:20.417 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:20 smithi029 ceph-mon[30044]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:20.856 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$3vcmRh6hX1wWqX6ZmLwdMOPKAy0BoGD1KAph3VwKHSmxY6aP8pGgK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1711686380, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-29T04:26:20.856 INFO:teuthology.orchestra.run.smithi029.stdout:Fetching dashboard port number... 2024-03-29T04:26:21.395 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:22.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:22 smithi029 ceph-mon[30044]: mgrmap e13: y(active, since 6s) 2024-03-29T04:26:22.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:22 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1341875838' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-29T04:26:22.186 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stdout 8443 2024-03-29T04:26:22.197 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:26:22.197 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout disabled 2024-03-29T04:26:22.204 INFO:teuthology.orchestra.run.smithi029.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:26:22.204 INFO:teuthology.orchestra.run.smithi029.stdout:systemctl: stdout inactive 2024-03-29T04:26:22.204 INFO:teuthology.orchestra.run.smithi029.stdout:firewalld.service is not enabled 2024-03-29T04:26:22.204 INFO:teuthology.orchestra.run.smithi029.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-29T04:26:22.206 INFO:teuthology.orchestra.run.smithi029.stdout:Ceph Dashboard is now available at: 2024-03-29T04:26:22.206 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:22.206 INFO:teuthology.orchestra.run.smithi029.stdout: URL: https://smithi029.front.sepia.ceph.com:8443/ 2024-03-29T04:26:22.206 INFO:teuthology.orchestra.run.smithi029.stdout: User: admin 2024-03-29T04:26:22.206 INFO:teuthology.orchestra.run.smithi029.stdout: Password: ppin15a8xd 2024-03-29T04:26:22.207 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:22.207 INFO:teuthology.orchestra.run.smithi029.stdout:Saving cluster configuration to /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config directory 2024-03-29T04:26:22.207 INFO:teuthology.orchestra.run.smithi029.stdout:Enabling autotune for osd_memory_target 2024-03-29T04:26:24.849 INFO:teuthology.orchestra.run.smithi029.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout:Or, if you are only running a single cluster on this host: 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.850 INFO:teuthology.orchestra.run.smithi029.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout:Please consider enabling telemetry to help improve Ceph: 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout: ceph telemetry on 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout:For more information see: 2024-03-29T04:26:24.851 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.852 INFO:teuthology.orchestra.run.smithi029.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-29T04:26:24.852 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:24.852 INFO:teuthology.orchestra.run.smithi029.stdout:Bootstrap complete. 2024-03-29T04:26:24.889 INFO:tasks.cephadm:Fetching config... 2024-03-29T04:26:24.890 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:26:24.890 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-29T04:26:24.906 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-29T04:26:24.906 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:26:24.906 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-29T04:26:24.961 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-29T04:26:24.961 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:26:24.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/keyring of=/dev/stdout 2024-03-29T04:26:25.028 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-29T04:26:25.028 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:26:25.028 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-29T04:26:25.083 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-29T04:26:25.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgC9dpq1PBP+ZLEEsYPg6unae/5cu1dVNrHOZL4A0p32vICXJOazQsotl9O3SczfE3jhSZ7j+3a2Oz81ZtTe92YUX2D0OsTstx8qt5wWjzj4ojbQNGroSF8sf13HNzugvnEIkzCeZsW8tetldqN2s473IeHHDzVTeqsRuFtoHho5+d1EC3/jeuax4rr1l1a2LU+3H7c4f7x40kd+Ja4SsRkl+JdyywgXl3yKaSuBkzfrhosrgCppPTDXaS1DSOMl7e8be+88CjaaS5ohzRVhp/gds4kBLso7L8oGrvg3ZVDIviuoB8O77AKSfslEOYhtdImH9dxDENAw/E3H+3y8Ly2FXsZzPtK/RTAkl28iCw4qXzU8hVGIu47tqLLQNTGyOA8PGgn1MaE8rxO3o06zl4Ipc7k7lq/N2X78nur3cqIUFWRVYoLtOlXYcBFUzfog6pp2HQPESkE/wVC2m7nanIWiDHE+Gmm67i6hZg4gL9TnyUtDnCsCaxmhIFPPvfJs8= ceph-37319ec8-ed84-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-29T04:26:25.180 INFO:teuthology.orchestra.run.smithi029.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgC9dpq1PBP+ZLEEsYPg6unae/5cu1dVNrHOZL4A0p32vICXJOazQsotl9O3SczfE3jhSZ7j+3a2Oz81ZtTe92YUX2D0OsTstx8qt5wWjzj4ojbQNGroSF8sf13HNzugvnEIkzCeZsW8tetldqN2s473IeHHDzVTeqsRuFtoHho5+d1EC3/jeuax4rr1l1a2LU+3H7c4f7x40kd+Ja4SsRkl+JdyywgXl3yKaSuBkzfrhosrgCppPTDXaS1DSOMl7e8be+88CjaaS5ohzRVhp/gds4kBLso7L8oGrvg3ZVDIviuoB8O77AKSfslEOYhtdImH9dxDENAw/E3H+3y8Ly2FXsZzPtK/RTAkl28iCw4qXzU8hVGIu47tqLLQNTGyOA8PGgn1MaE8rxO3o06zl4Ipc7k7lq/N2X78nur3cqIUFWRVYoLtOlXYcBFUzfog6pp2HQPESkE/wVC2m7nanIWiDHE+Gmm67i6hZg4gL9TnyUtDnCsCaxmhIFPPvfJs8= ceph-37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:26:25.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgC9dpq1PBP+ZLEEsYPg6unae/5cu1dVNrHOZL4A0p32vICXJOazQsotl9O3SczfE3jhSZ7j+3a2Oz81ZtTe92YUX2D0OsTstx8qt5wWjzj4ojbQNGroSF8sf13HNzugvnEIkzCeZsW8tetldqN2s473IeHHDzVTeqsRuFtoHho5+d1EC3/jeuax4rr1l1a2LU+3H7c4f7x40kd+Ja4SsRkl+JdyywgXl3yKaSuBkzfrhosrgCppPTDXaS1DSOMl7e8be+88CjaaS5ohzRVhp/gds4kBLso7L8oGrvg3ZVDIviuoB8O77AKSfslEOYhtdImH9dxDENAw/E3H+3y8Ly2FXsZzPtK/RTAkl28iCw4qXzU8hVGIu47tqLLQNTGyOA8PGgn1MaE8rxO3o06zl4Ipc7k7lq/N2X78nur3cqIUFWRVYoLtOlXYcBFUzfog6pp2HQPESkE/wVC2m7nanIWiDHE+Gmm67i6hZg4gL9TnyUtDnCsCaxmhIFPPvfJs8= ceph-37319ec8-ed84-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-29T04:26:25.241 INFO:teuthology.orchestra.run.smithi099.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgC9dpq1PBP+ZLEEsYPg6unae/5cu1dVNrHOZL4A0p32vICXJOazQsotl9O3SczfE3jhSZ7j+3a2Oz81ZtTe92YUX2D0OsTstx8qt5wWjzj4ojbQNGroSF8sf13HNzugvnEIkzCeZsW8tetldqN2s473IeHHDzVTeqsRuFtoHho5+d1EC3/jeuax4rr1l1a2LU+3H7c4f7x40kd+Ja4SsRkl+JdyywgXl3yKaSuBkzfrhosrgCppPTDXaS1DSOMl7e8be+88CjaaS5ohzRVhp/gds4kBLso7L8oGrvg3ZVDIviuoB8O77AKSfslEOYhtdImH9dxDENAw/E3H+3y8Ly2FXsZzPtK/RTAkl28iCw4qXzU8hVGIu47tqLLQNTGyOA8PGgn1MaE8rxO3o06zl4Ipc7k7lq/N2X78nur3cqIUFWRVYoLtOlXYcBFUzfog6pp2HQPESkE/wVC2m7nanIWiDHE+Gmm67i6hZg4gL9TnyUtDnCsCaxmhIFPPvfJs8= ceph-37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:26:25.257 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-29T04:26:25.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:25 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3106878737' entity='client.admin' 2024-03-29T04:26:25.514 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:26:28.091 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:28.091 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:28.091 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi029", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:26:28.091 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:28.091 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:28.092 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:29.049 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-29T04:26:29.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-29T04:26:29.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:29 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1553337301' entity='client.admin' 2024-03-29T04:26:29.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:29 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:29.657 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:29 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:29.660 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:26:32.328 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi099 2024-03-29T04:26:32.328 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:26:32.329 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/etc/ceph/ceph.conf 2024-03-29T04:26:32.347 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:26:32.347 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:26:32.404 INFO:tasks.cephadm:Adding host smithi099 to orchestrator... 2024-03-29T04:26:32.404 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch host add smithi099 2024-03-29T04:26:32.667 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:26:32.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:32.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:32.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:32.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:32.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:32.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:26:32.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:26:32.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:26:32.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:32.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:32.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:32 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:33.765 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:33 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:26:36.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:35 smithi029 ceph-mon[30044]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi099", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:36.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:35 smithi029 ceph-mon[30044]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:36.725 INFO:teuthology.orchestra.run.smithi029.stdout:Added host 'smithi099' with addr '172.21.15.99' 2024-03-29T04:26:37.035 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:36 smithi029 ceph-mon[30044]: Deploying cephadm binary to smithi099 2024-03-29T04:26:37.336 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch host ls --format=json 2024-03-29T04:26:37.593 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:26:37.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:37 smithi029 ceph-mon[30044]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:37.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:37.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:37 smithi029 ceph-mon[30044]: Added host smithi099 2024-03-29T04:26:37.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:37.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:37.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:39.400 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:26:39.401 INFO:teuthology.orchestra.run.smithi029.stdout:[{"addr": "172.21.15.29", "hostname": "smithi029", "labels": [], "status": ""}, {"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}] 2024-03-29T04:26:39.401 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:39.401 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:39 smithi029 ceph-mon[30044]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:40.011 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-29T04:26:40.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd crush tunables default 2024-03-29T04:26:40.263 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:26:40.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:40 smithi029 ceph-mon[30044]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:26:41.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:41 smithi029 ceph-mon[30044]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:42.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:26:42.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:42.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:42.420 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:43.142 INFO:teuthology.orchestra.run.smithi029.stderr:adjusted tunables profile to default 2024-03-29T04:26:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:43 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2311031119' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-29T04:26:43.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:43 smithi029 ceph-mon[30044]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:43.737 INFO:tasks.cephadm:Adding mon.a on smithi029 2024-03-29T04:26:43.737 INFO:tasks.cephadm:Adding mon.c on smithi029 2024-03-29T04:26:43.737 INFO:tasks.cephadm:Adding mon.b on smithi099 2024-03-29T04:26:43.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch apply mon '3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b' 2024-03-29T04:26:43.985 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:26:44.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:44 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2311031119' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-29T04:26:44.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:44 smithi029 ceph-mon[30044]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:26:45.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:45 smithi029 ceph-mon[30044]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:45.850 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled mon update... 2024-03-29T04:26:46.352 DEBUG:teuthology.orchestra.run.smithi029:mon.c> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.c.service 2024-03-29T04:26:46.358 DEBUG:teuthology.orchestra.run.smithi099:mon.b> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.b.service 2024-03-29T04:26:46.360 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:26:46.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:26:47.143 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b;count:3 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:47.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:46 smithi029 ceph-mon[30044]: Deploying daemon mon.b on smithi099 2024-03-29T04:26:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:47 smithi029 ceph-mon[30044]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:50.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:49 smithi029 ceph-mon[30044]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:50.516 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:26:50.517 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":1,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","modified":"2024-03-29T04:24:58.066690Z","created":"2024-03-29T04:24:58.066690Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-29T04:26:50.518 INFO:teuthology.orchestra.run.smithi099.stderr:dumped monmap epoch 1 2024-03-29T04:26:50.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 podman[33815]: 2024-03-29 04:26:50.654813069 +0000 UTC m=+0.359217160 container init 48c38660bc126f3622d55eb30e8fc187899212ed7d0ed3506868c15be8bb4ec1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-29T04:26:50.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 podman[33815]: 2024-03-29 04:26:50.66081274 +0000 UTC m=+0.365216817 container start 48c38660bc126f3622d55eb30e8fc187899212ed7d0ed3506868c15be8bb4ec1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable), process ceph-mon, pid 7 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: pidfile_write: ignore empty --pid-file 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: load: jerasure load: lrc 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: RocksDB version: 6.15.5 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Compile date Mar 28 2024 2024-03-29T04:26:51.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: DB SUMMARY 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: DB Session ID: E7LZGF7G5QXZPI41QXJ7 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.env: 0x559dbcf66a80 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.fs: Posix File System 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.info_log: 0x559dbf01ae80 2024-03-29T04:26:51.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.statistics: (nil) 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.use_fsync: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.db_log_dir: 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-03-29T04:26:51.020 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:26:51.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:26:51.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:26:51.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:26:51.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:26:51.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:26:51.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.write_buffer_manager: 0x559dbfd8a960 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:26:51.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.unordered_write: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.row_cache: None 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.wal_filter: None 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.preserve_deletes: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:26:51.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:26:51.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_open_files: -1 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:26:51.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Compression algorithms supported: 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kZSTD supported: 0 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.merge_operator: 2024-03-29T04:26:51.027 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_filter: None 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559dbf06af58) 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_type: 0 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_index_type: 0 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_shortening: 1 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:26:51.028 INFO:journalctl@ceph.mon.b.smithi099.stdout: hash_index_allow_collision: 1 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: checksum: 1 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: no_block_cache: 0 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache: 0x559dbf09d090 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_options: 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: capacity : 536870912 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_shard_bits : 4 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: strict_capacity_limit : 0 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_compressed: (nil) 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: persistent_cache: (nil) 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size: 4096 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size_deviation: 10 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_restart_interval: 16 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_block_restart_interval: 1 2024-03-29T04:26:51.029 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_block_size: 4096 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: partition_filters: 0 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: use_delta_encoding: 1 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: whole_key_filtering: 1 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: verify_compression: 0 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: format_version: 4 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_index_compression: 1 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_align: 0 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression: NoCompression 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:26:51.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.num_levels: 7 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:26:51.031 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:26:51.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.arena_block_size: 4194304 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:26:51.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:26:51.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.ttl: 2592000 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:26:51.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686410687720, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686410688023, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711686410, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "79578a7e-8e49-4156-a6e0-c4537304156e", "db_session_id": "E7LZGF7G5QXZPI41QXJ7"}} 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686410688331, "job": 1, "event": "recovery_finished"} 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x559dbf031180 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: DB pointer 0x559dbf106000 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** DB Stats ** 2024-03-29T04:26:51.036 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.037 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:26:51.038 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:51.039 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: using public_addr v2:172.21.15.99:0/0 -> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: starting mon.b rank -1 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:26:51.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(???) e0 preinit fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).mds e1 new map 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).mds e1 print_map 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout: e1 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout: legacy client fscid: -1 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout: No filesystems configured 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-29T04:26:51.041 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:26:51.042 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2311031119' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2311031119' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b;count:3 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:51.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: Deploying daemon mon.b on smithi099 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 ceph-mon[33865]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 bash[33815]: 48c38660bc126f3622d55eb30e8fc187899212ed7d0ed3506868c15be8bb4ec1 2024-03-29T04:26:51.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:50 smithi099 systemd[1]: Started Ceph mon.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:26:52.521 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:26:52.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:26:52.774 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:26:55.978 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:26:55.978 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":2,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","modified":"2024-03-29T04:26:50.708941Z","created":"2024-03-29T04:24:58.066690Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-29T04:26:55.980 INFO:teuthology.orchestra.run.smithi099.stderr:dumped monmap epoch 2 2024-03-29T04:26:56.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:26:56.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: mon.a calling monitor election 2024-03-29T04:26:56.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:56.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: mon.b calling monitor election 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: fsmap 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: mgrmap e13: y(active, since 41s) 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: overall HEALTH_OK 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:56.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:56.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:56.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:26:56.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:55 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: mon.a calling monitor election 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: mon.b calling monitor election 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:56.159 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: fsmap 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: mgrmap e13: y(active, since 41s) 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: overall HEALTH_OK 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:26:56.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:55 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:56.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:56 smithi099 ceph-mon[33865]: Deploying daemon mon.c on smithi029 2024-03-29T04:26:56.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:56 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/2440632787' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:26:56.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:56 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:56.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:56 smithi029 ceph-mon[30044]: Deploying daemon mon.c on smithi029 2024-03-29T04:26:56.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:56 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/2440632787' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:26:56.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:56 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:57.556 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:26:57.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:26:57.812 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:26:57.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:26:57 smithi099 ceph-mon[33865]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:57.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:26:57 smithi029 ceph-mon[30044]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.147 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 podman[41861]: 2024-03-29 04:26:58.94419835 +0000 UTC m=+0.017751696 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:26:59.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 podman[41861]: 2024-03-29 04:26:59.252535187 +0000 UTC m=+0.326088514 container init 0159b319c9a57c9626417cca5eea6936443cb5ce21f611579877be1d6b215b29 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-29T04:26:59.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 podman[41861]: 2024-03-29 04:26:59.25818805 +0000 UTC m=+0.331741377 container start 0159b319c9a57c9626417cca5eea6936443cb5ce21f611579877be1d6b215b29 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable), process ceph-mon, pid 7 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pidfile_write: ignore empty --pid-file 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: load: jerasure load: lrc 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: RocksDB version: 6.15.5 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Compile date Mar 28 2024 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: DB SUMMARY 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: DB Session ID: RZUZN9HM1DWP3XHI9W79 2024-03-29T04:26:59.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:26:59.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.env: 0x560c43293a80 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.fs: Posix File System 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.info_log: 0x560c45480e80 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.statistics: (nil) 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.use_fsync: 0 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:26:59.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.db_log_dir: 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:26:59.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.write_buffer_manager: 0x560c461f0930 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:26:59.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.unordered_write: 0 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.row_cache: None 2024-03-29T04:26:59.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.wal_filter: None 2024-03-29T04:26:59.676 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:26:59.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:26:59.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.preserve_deletes: 0 2024-03-29T04:26:59.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:26:59.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:26:59.677 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:26:59.678 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_open_files: -1 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:26:59.679 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Compression algorithms supported: 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kZSTD supported: 0 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.merge_operator: 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_filter: None 2024-03-29T04:26:59.680 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560c454d0f58) 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_type: 0 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_index_type: 0 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_shortening: 1 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: hash_index_allow_collision: 1 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: checksum: 1 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: no_block_cache: 0 2024-03-29T04:26:59.681 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache: 0x560c45503090 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_options: 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: capacity : 536870912 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_shard_bits : 4 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: strict_capacity_limit : 0 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_compressed: (nil) 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: persistent_cache: (nil) 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size: 4096 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size_deviation: 10 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_restart_interval: 16 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_block_restart_interval: 1 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_block_size: 4096 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: partition_filters: 0 2024-03-29T04:26:59.682 INFO:journalctl@ceph.mon.c.smithi029.stdout: use_delta_encoding: 1 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: whole_key_filtering: 1 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: verify_compression: 0 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: format_version: 4 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_index_compression: 1 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_align: 0 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression: NoCompression 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.num_levels: 7 2024-03-29T04:26:59.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:26:59.684 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:26:59.685 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.arena_block_size: 4194304 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:26:59.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:26:59.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.ttl: 2592000 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686419285442, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-03-29T04:26:59.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686419285767, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711686419, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "78409c6f-4e62-4224-ad72-b11bdd045aa1", "db_session_id": "RZUZN9HM1DWP3XHI9W79"}} 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686419286113, "job": 1, "event": "recovery_finished"} 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560c45497180 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: DB pointer 0x560c4556c000 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** DB Stats ** 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:26:59.689 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:26:59.690 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:59.691 INFO:journalctl@ceph.mon.c.smithi029.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:26:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: using public_addrv [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: starting mon.c rank -1 at public addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] at bind addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(???) e0 preinit fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).mds e1 new map 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).mds e1 print_map 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: e1 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: legacy client fscid: -1 2024-03-29T04:26:59.693 INFO:journalctl@ceph.mon.c.smithi029.stdout: 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout: No filesystems configured 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2311031119' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2311031119' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Saving service mon spec with placement smithi029:172.21.15.29=a;smithi029:[v2:172.21.15.29:3301,v1:172.21.15.29:6790]=c;smithi099:172.21.15.99=b;count:3 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Deploying daemon mon.b on smithi099 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.a calling monitor election 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.b calling monitor election 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: monmap e2: 2 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: fsmap 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mgrmap e13: y(active, since 41s) 2024-03-29T04:26:59.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: overall HEALTH_OK 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: Deploying daemon mon.c on smithi029 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/2440632787' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 ceph-mon[41899]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 bash[41861]: 0159b319c9a57c9626417cca5eea6936443cb5ce21f611579877be1d6b215b29 2024-03-29T04:26:59.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:26:59 smithi029 systemd[1]: Started Ceph mon.c for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: mon.a calling monitor election 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: mon.b calling monitor election 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: mon.c calling monitor election 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: fsmap 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: mgrmap e13: y(active, since 49s) 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: overall HEALTH_OK 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: mon.a calling monitor election 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: mon.b calling monitor election 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: mon.c calling monitor election 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: fsmap 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: mgrmap e13: y(active, since 49s) 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: overall HEALTH_OK 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: mon.a calling monitor election 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: mon.b calling monitor election 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: mon.c calling monitor election 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: monmap e3: 3 mons at {a=[v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: fsmap 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: mgrmap e13: y(active, since 49s) 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: overall HEALTH_OK 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:04.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:05.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:05.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:05 smithi099 ceph-mon[33865]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:05.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:05.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:05.604 INFO:teuthology.orchestra.run.smithi099.stdout: 2024-03-29T04:27:05.604 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":3,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","modified":"2024-03-29T04:26:59.304920Z","created":"2024-03-29T04:24:58.066690Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3300","nonce":0},{"type":"v1","addr":"172.21.15.29:6789","nonce":0}]},"addr":"172.21.15.29:6789/0","public_addr":"172.21.15.29:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:3301","nonce":0},{"type":"v1","addr":"172.21.15.29:6790","nonce":0}]},"addr":"172.21.15.29:6790/0","public_addr":"172.21.15.29:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-03-29T04:27:05.605 INFO:teuthology.orchestra.run.smithi099.stderr:dumped monmap epoch 3 2024-03-29T04:27:05.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:05.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[30044]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:05.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:05.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:05.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:05.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[41899]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:05.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:05.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:27:06.271 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-29T04:27:06.271 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph config generate-minimal-conf 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/3131727696' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:27:06.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/3131727696' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:06.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:06.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/3131727696' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:06.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:06.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:06.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:07.644 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:07 smithi029 ceph-mon[41899]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:27:07.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:07 smithi029 ceph-mon[41899]: Reconfiguring daemon mon.a on smithi029 2024-03-29T04:27:07.645 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:07 smithi029 ceph-mon[41899]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:07.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:07 smithi029 ceph-mon[30044]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:27:07.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:07 smithi029 ceph-mon[30044]: Reconfiguring daemon mon.a on smithi029 2024-03-29T04:27:07.645 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:07 smithi029 ceph-mon[30044]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:07.728 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:27:07.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:07 smithi099 ceph-mon[33865]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:27:07.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:07 smithi099 ceph-mon[33865]: Reconfiguring daemon mon.a on smithi029 2024-03-29T04:27:07.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:07 smithi099 ceph-mon[33865]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:08.823 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[30044]: Reconfiguring daemon mon.c on smithi029 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:08.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:08.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:08.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:08 smithi029 ceph-mon[41899]: Reconfiguring daemon mon.c on smithi029 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:09.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:08 smithi099 ceph-mon[33865]: Reconfiguring daemon mon.c on smithi029 2024-03-29T04:27:09.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:09 smithi029 ceph-mon[30044]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:09.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:09 smithi029 ceph-mon[41899]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:10.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:09 smithi099 ceph-mon[33865]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:10.724 INFO:teuthology.orchestra.run.smithi029.stdout:# minimal ceph.conf for 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:27:10.724 INFO:teuthology.orchestra.run.smithi029.stdout:[global] 2024-03-29T04:27:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: fsid = 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:27:10.725 INFO:teuthology.orchestra.run.smithi029.stdout: mon_host = [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] 2024-03-29T04:27:11.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: Reconfiguring daemon mon.b on smithi099 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:11.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1198986989' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: Reconfiguring daemon mon.b on smithi099 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:11.332 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:11 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1198986989' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:11.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: Reconfiguring daemon mon.b on smithi099 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:11.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:11 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1198986989' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:11.359 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-29T04:27:11.359 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:27:11.359 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.conf 2024-03-29T04:27:11.392 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:27:11.392 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:27:11.460 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:27:11.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.conf 2024-03-29T04:27:11.491 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:27:11.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:27:11.558 INFO:tasks.cephadm:Adding mgr.y on smithi029 2024-03-29T04:27:11.558 INFO:tasks.cephadm:Adding mgr.x on smithi099 2024-03-29T04:27:11.559 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch apply mgr '2;smithi029=y;smithi099=x' 2024-03-29T04:27:11.838 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:27:12.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:12.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:12.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:12.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:12.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.438 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled mgr update... 2024-03-29T04:27:13.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:13 smithi099 ceph-mon[33865]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:13.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:13 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:13 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:13.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:13 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:13.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:13 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:13.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:13 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[30044]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[41899]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:13.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:13 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:13.997 DEBUG:teuthology.orchestra.run.smithi099:mgr.x> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service 2024-03-29T04:27:14.000 INFO:tasks.cephadm:Deploying OSDs... 2024-03-29T04:27:14.000 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:27:14.000 DEBUG:teuthology.orchestra.run.smithi029:> dd if=/scratch_devs of=/dev/stdout 2024-03-29T04:27:14.018 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-29T04:27:14.019 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_1 2024-03-29T04:27:14.076 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-03-29 04:26:58.767012924 +0000 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-03-29 04:21:32.974072686 +0000 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-03-29 04:21:32.974072686 +0000 2024-03-29T04:27:14.077 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-03-29 04:21:32.974072686 +0000 2024-03-29T04:27:14.077 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-29T04:27:14.146 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-03-29T04:27:14.147 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-03-29T04:27:14.147 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000123269 s, 4.2 MB/s 2024-03-29T04:27:14.148 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-29T04:27:14.206 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_2 2024-03-29T04:27:14.264 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-29T04:27:14.264 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:14.264 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-03-29T04:27:14.264 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:14.264 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:14.264 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-03-29 04:26:58.767012924 +0000 2024-03-29T04:27:14.265 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-03-29 04:21:33.316066468 +0000 2024-03-29T04:27:14.265 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-03-29 04:21:33.316066468 +0000 2024-03-29T04:27:14.265 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-03-29 04:21:33.316066468 +0000 2024-03-29T04:27:14.265 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-29T04:27:14.331 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-03-29T04:27:14.332 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-03-29T04:27:14.332 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000116437 s, 4.4 MB/s 2024-03-29T04:27:14.333 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-29T04:27:14.390 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_3 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-03-29 04:26:58.768012906 +0000 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-03-29 04:21:33.654060322 +0000 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-03-29 04:21:33.654060322 +0000 2024-03-29T04:27:14.446 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-03-29 04:21:33.654060322 +0000 2024-03-29T04:27:14.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-29T04:27:14.514 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-03-29T04:27:14.515 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-03-29T04:27:14.515 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 0.000104596 s, 4.9 MB/s 2024-03-29T04:27:14.516 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-29T04:27:14.572 DEBUG:teuthology.orchestra.run.smithi029:> stat /dev/vg_nvme/lv_4 2024-03-29T04:27:14.627 INFO:teuthology.orchestra.run.smithi029.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-29T04:27:14.627 INFO:teuthology.orchestra.run.smithi029.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:14.627 INFO:teuthology.orchestra.run.smithi029.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-03-29T04:27:14.627 INFO:teuthology.orchestra.run.smithi029.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:14.628 INFO:teuthology.orchestra.run.smithi029.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:14.628 INFO:teuthology.orchestra.run.smithi029.stdout:Access: 2024-03-29 04:26:58.768012906 +0000 2024-03-29T04:27:14.628 INFO:teuthology.orchestra.run.smithi029.stdout:Modify: 2024-03-29 04:21:33.993054158 +0000 2024-03-29T04:27:14.628 INFO:teuthology.orchestra.run.smithi029.stdout:Change: 2024-03-29 04:21:33.993054158 +0000 2024-03-29T04:27:14.628 INFO:teuthology.orchestra.run.smithi029.stdout: Birth: 2024-03-29 04:21:33.993054158 +0000 2024-03-29T04:27:14.628 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-29T04:27:14.694 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records in 2024-03-29T04:27:14.695 INFO:teuthology.orchestra.run.smithi029.stderr:1+0 records out 2024-03-29T04:27:14.695 INFO:teuthology.orchestra.run.smithi029.stderr:512 bytes copied, 9.4703e-05 s, 5.4 MB/s 2024-03-29T04:27:14.696 DEBUG:teuthology.orchestra.run.smithi029:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-29T04:27:14.752 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:27:14.752 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/scratch_devs of=/dev/stdout 2024-03-29T04:27:14.767 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-29T04:27:14.768 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_1 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-29 04:26:49.987866403 +0000 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-29 04:21:25.537337718 +0000 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-29 04:21:25.537337718 +0000 2024-03-29T04:27:14.823 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-29 04:21:25.537337718 +0000 2024-03-29T04:27:14.823 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-29T04:27:14.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi099=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:14.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: Saving service mgr spec with placement smithi029=y;smithi099=x;count:2 2024-03-29T04:27:14.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:27:14.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:27:14.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:27:14.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:14.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:14 smithi099 ceph-mon[33865]: Deploying daemon mgr.x on smithi099 2024-03-29T04:27:14.875 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-29T04:27:14.875 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-29T04:27:14.875 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000216902 s, 2.4 MB/s 2024-03-29T04:27:14.876 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi099=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: Saving service mgr spec with placement smithi029=y;smithi099=x;count:2 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[30044]: Deploying daemon mgr.x on smithi099 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi029=y;smithi099=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: Saving service mgr spec with placement smithi029=y;smithi099=x;count:2 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:14 smithi029 ceph-mon[41899]: Deploying daemon mgr.x on smithi099 2024-03-29T04:27:14.932 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_2 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-29 04:26:49.987866403 +0000 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-29 04:21:25.890329513 +0000 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-29 04:21:25.890329513 +0000 2024-03-29T04:27:14.987 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-29 04:21:25.890329513 +0000 2024-03-29T04:27:14.988 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-29T04:27:15.052 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-29T04:27:15.052 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-29T04:27:15.052 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000217867 s, 2.4 MB/s 2024-03-29T04:27:15.053 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-29T04:27:15.109 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_3 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-29 04:26:49.987866403 +0000 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-29 04:21:26.243321308 +0000 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-29 04:21:26.243321308 +0000 2024-03-29T04:27:15.165 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-29 04:21:26.243321308 +0000 2024-03-29T04:27:15.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-29T04:27:15.230 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-29T04:27:15.230 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-29T04:27:15.230 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000233985 s, 2.2 MB/s 2024-03-29T04:27:15.231 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-29T04:27:15.287 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_4 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2024-03-29 04:26:49.988866380 +0000 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2024-03-29 04:21:26.565313823 +0000 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2024-03-29 04:21:26.565313823 +0000 2024-03-29T04:27:15.343 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: 2024-03-29 04:21:26.565313823 +0000 2024-03-29T04:27:15.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-29T04:27:15.426 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2024-03-29T04:27:15.426 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2024-03-29T04:27:15.427 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000171877 s, 3.0 MB/s 2024-03-29T04:27:15.427 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-29T04:27:15.483 INFO:tasks.cephadm:Deploying osd.0 on smithi029 with /dev/vg_nvme/lv_4... 2024-03-29T04:27:15.484 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-29T04:27:15.747 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:27:15.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:15 smithi099 ceph-mon[33865]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:15.842 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:15 smithi099 systemd[1]: Starting Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:27:15.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:15 smithi029 ceph-mon[30044]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:15.869 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:15 smithi029 ceph-mon[41899]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:16.098 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:15 smithi099 podman[35969]: 2024-03-29 04:27:15.865860855 +0000 UTC m=+0.111448221 container create 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:27:16.099 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:15 smithi099 podman[35969]: 2024-03-29 04:27:15.771172714 +0000 UTC m=+0.016760026 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:27:16.099 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 podman[35969]: 2024-03-29 04:27:16.085047785 +0000 UTC m=+0.330635095 container init 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-29T04:27:16.099 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 podman[35969]: 2024-03-29 04:27:16.090901069 +0000 UTC m=+0.336488393 container start 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=quincy-4213e17) 2024-03-29T04:27:16.099 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 ceph-mgr[36007]: -- 172.21.15.99:0/1230160117 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558adc4929c0 con 0x558adc468800 2024-03-29T04:27:16.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 bash[35969]: 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b 2024-03-29T04:27:16.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 systemd[1]: Started Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:27:16.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:16.615+0000 7f01a6d491c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:27:16.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:16.715+0000 7f01a6d491c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:27:17.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[41899]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:17 smithi029 ceph-mon[30044]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-mon[33865]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:17.372+0000 7f01a6d491c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:17.462+0000 7f01a6d491c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:27:17.524 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:17.547+0000 7f01a6d491c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:27:17.749 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-29T04:27:17.750 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-29T04:27:17.750 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-03-29T04:27:17.750 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-03-29T04:27:17.750 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00887611 s, 1.2 GB/s 2024-03-29T04:27:17.750 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-03-29T04:27:17.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:17.694+0000 7f01a6d491c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:27:17.849 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:17.771+0000 7f01a6d491c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:27:18.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:17.938+0000 7f01a6d491c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:27:18.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:18.117+0000 7f01a6d491c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:27:18.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:18.197+0000 7f01a6d491c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:27:18.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:18.287+0000 7f01a6d491c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:27:18.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:18.370+0000 7f01a6d491c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:18.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:18 smithi099 ceph-mon[33865]: Reconfiguring daemon mgr.y on smithi029 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:18.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[41899]: Reconfiguring daemon mgr.y on smithi029 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:18.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:18 smithi029 ceph-mon[30044]: Reconfiguring daemon mgr.y on smithi029 2024-03-29T04:27:19.075 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi029:vg_nvme/lv_4 2024-03-29T04:27:19.573 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:27:19.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:19 smithi029 ceph-mon[41899]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:19.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:19 smithi029 ceph-mon[30044]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:19.693 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:19 smithi099 ceph-mon[33865]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:19.693 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:19.401+0000 7f01a6d491c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:27:19.693 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:19.529+0000 7f01a6d491c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:27:19.693 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:19.716+0000 7f01a6d491c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:27:20.090 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:19.942+0000 7f01a6d491c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:27:20.090 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:20.113+0000 7f01a6d491c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:27:20.348 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:20.290+0000 7f01a6d491c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:27:20.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:20.543+0000 7f01a6d491c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:27:20.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:20.626+0000 7f01a6d491c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.867 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:20.868 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:20.869 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:21.348 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:21.008+0000 7f01a6d491c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:27:21.831 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[30044]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:21.831 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:27:21.831 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:27:21.831 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:21.831 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[41899]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:21.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:27:21.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:27:21.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:21.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:21.770+0000 7f01a6d491c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:27:21.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:21 smithi099 ceph-mon[33865]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:21.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:27:21.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:27:21.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:22.343 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:22.058+0000 7f01a6d491c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:27:22.343 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:22.366+0000 7f01a6d491c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:27:22.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:27:22.553+0000 7f01a6d491c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:27:22.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-mon[33865]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:22.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-mon[33865]: Standby manager daemon x started 2024-03-29T04:27:22.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:27:22.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:27:22.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:27:22.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:22 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:27:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[30044]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[30044]: Standby manager daemon x started 2024-03-29T04:27:22.820 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[41899]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[41899]: Standby manager daemon x started 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:27:22.821 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:22 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2101936081' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:27:23.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:23 smithi099 ceph-mon[33865]: mgrmap e14: y(active, since 67s), standbys: x 2024-03-29T04:27:23.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:23 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:27:23.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:23 smithi099 ceph-mon[33865]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:23.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:23 smithi029 ceph-mon[30044]: mgrmap e14: y(active, since 67s), standbys: x 2024-03-29T04:27:23.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:23 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:27:23.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:23 smithi029 ceph-mon[30044]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:23.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:23 smithi029 ceph-mon[41899]: mgrmap e14: y(active, since 67s), standbys: x 2024-03-29T04:27:23.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:23 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:27:23.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:23 smithi029 ceph-mon[41899]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:24.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:24 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1454316886' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65937eeb-6b63-4697-8961-81cdfe926a8a"}]: dispatch 2024-03-29T04:27:24.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:24 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1454316886' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "65937eeb-6b63-4697-8961-81cdfe926a8a"}]': finished 2024-03-29T04:27:24.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:24 smithi099 ceph-mon[33865]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:27:24.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:24 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:24.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:24 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3234305612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:27:24.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1454316886' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65937eeb-6b63-4697-8961-81cdfe926a8a"}]: dispatch 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1454316886' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "65937eeb-6b63-4697-8961-81cdfe926a8a"}]': finished 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[30044]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3234305612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1454316886' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "65937eeb-6b63-4697-8961-81cdfe926a8a"}]: dispatch 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1454316886' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "65937eeb-6b63-4697-8961-81cdfe926a8a"}]': finished 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[41899]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:24.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:24 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3234305612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:27:25.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:25 smithi099 ceph-mon[33865]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:25.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:25 smithi029 ceph-mon[30044]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:25.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:25 smithi029 ceph-mon[41899]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:27.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:27 smithi029 ceph-mon[30044]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:27.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:27 smithi029 ceph-mon[41899]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:28.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:27 smithi099 ceph-mon[33865]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:30.058 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:29 smithi029 ceph-mon[30044]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:30.059 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:29 smithi029 ceph-mon[41899]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:30.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:29 smithi099 ceph-mon[33865]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:30.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:27:30.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:30.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:27:30.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:31.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:27:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:31.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:31 smithi029 ceph-mon[41899]: Deploying daemon osd.0 on smithi029 2024-03-29T04:27:31.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:31 smithi029 ceph-mon[41899]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:31.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:31 smithi029 ceph-mon[30044]: Deploying daemon osd.0 on smithi029 2024-03-29T04:27:31.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:31 smithi029 ceph-mon[30044]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:32.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:31 smithi099 ceph-mon[33865]: Deploying daemon osd.0 on smithi029 2024-03-29T04:27:32.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:31 smithi099 ceph-mon[33865]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:34.008 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:33 smithi029 ceph-mon[30044]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:34.009 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:33 smithi029 ceph-mon[41899]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:34.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:33 smithi099 ceph-mon[33865]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:36.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:35 smithi099 ceph-mon[33865]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:36.102 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:36.102 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:36.102 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:36.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[30044]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:36.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:36.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:36.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:36.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[41899]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:36.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:36.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:36.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:37.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:37 smithi029 ceph-mon[41899]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:37.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:37 smithi029 ceph-mon[30044]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:38.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:37 smithi099 ceph-mon[33865]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:39.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[30044]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:27:39.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:39 smithi029 ceph-mon[41899]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:27:39.424 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 0 on host 'smithi029' 2024-03-29T04:27:39.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:27:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:39 smithi099 ceph-mon[33865]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:27:40.082 DEBUG:teuthology.orchestra.run.smithi029:osd.0> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.0.service 2024-03-29T04:27:40.085 INFO:tasks.cephadm:Deploying osd.1 on smithi029 with /dev/vg_nvme/lv_3... 2024-03-29T04:27:40.085 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:40.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:40.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:40.393 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[46337]: 2024-03-29T04:27:40.320+0000 7f368c45c700 -1 osd.0 0 waiting for initial osdmap 2024-03-29T04:27:40.393 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:27:40 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[46337]: 2024-03-29T04:27:40.322+0000 7f3683a50700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:27:40.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:27:40.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:27:40.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:40.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:27:40.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:27:40.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:40.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:40.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:40 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:27:40.848 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[30044]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[30044]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[30044]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[41899]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[41899]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:27:41.567 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:41.568 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:41.568 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:41 smithi029 ceph-mon[41899]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:41.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:41 smithi099 ceph-mon[33865]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:27:41.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:41 smithi099 ceph-mon[33865]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:27:41.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:41 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:41.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:41 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:41.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:41 smithi099 ceph-mon[33865]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[30044]: osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633] boot 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[30044]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:42.412 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:27:42.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:27:42.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:42.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[41899]: osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633] boot 2024-03-29T04:27:42.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[41899]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:27:42.413 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:42 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:42.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:42 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:27:42.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:42 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:27:42.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:42 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:42.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:42 smithi099 ceph-mon[33865]: osd.0 [v2:172.21.15.29:6802/517562633,v1:172.21.15.29:6803/517562633] boot 2024-03-29T04:27:42.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:42 smithi099 ceph-mon[33865]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:27:42.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:42 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:27:43.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:43 smithi029 ceph-mon[30044]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:27:43.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:43 smithi029 ceph-mon[30044]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:43.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:43 smithi029 ceph-mon[41899]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:27:43.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:43 smithi029 ceph-mon[41899]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:43.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:43 smithi099 ceph-mon[33865]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:27:43.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:43 smithi099 ceph-mon[33865]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:43.926 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-29T04:27:43.926 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-29T04:27:43.926 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-03-29T04:27:43.926 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-03-29T04:27:43.926 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00840225 s, 1.2 GB/s 2024-03-29T04:27:43.926 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-03-29T04:27:44.651 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi029:vg_nvme/lv_3 2024-03-29T04:27:44.906 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:27:45.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi029 2024-03-29T04:27:45.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:27:45.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi029 to 16069M 2024-03-29T04:27:45.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.282 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:45.282 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:45.282 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.282 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[30044]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:45.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi029 2024-03-29T04:27:45.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:27:45.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi029 to 16069M 2024-03-29T04:27:45.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:45.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:45.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:45 smithi029 ceph-mon[41899]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:45.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi029 2024-03-29T04:27:45.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi029 to 16069M 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:27:45.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:45 smithi099 ceph-mon[33865]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:47.849 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[30044]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[30044]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[41899]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[41899]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:27:47.850 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:27:47.851 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:47 smithi099 ceph-mon[33865]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:48.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:47 smithi099 ceph-mon[33865]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:27:48.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:27:48.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:27:48.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:50.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:49 smithi099 ceph-mon[33865]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:50.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:49 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4140556867' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f30ee5e-a787-4f20-a79a-27badee86b8b"}]: dispatch 2024-03-29T04:27:50.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:49 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4140556867' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f30ee5e-a787-4f20-a79a-27badee86b8b"}]': finished 2024-03-29T04:27:50.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:49 smithi099 ceph-mon[33865]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:27:50.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[30044]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4140556867' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f30ee5e-a787-4f20-a79a-27badee86b8b"}]: dispatch 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4140556867' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f30ee5e-a787-4f20-a79a-27badee86b8b"}]': finished 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[30044]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[41899]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4140556867' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f30ee5e-a787-4f20-a79a-27badee86b8b"}]: dispatch 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4140556867' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f30ee5e-a787-4f20-a79a-27badee86b8b"}]': finished 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[41899]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:27:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:27:51.095 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:50 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1042544907' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:27:51.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:50 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1042544907' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:27:51.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:50 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1042544907' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:27:52.052 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:51 smithi029 ceph-mon[30044]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:52.052 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:51 smithi029 ceph-mon[41899]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:52.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:51 smithi099 ceph-mon[33865]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:54.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:53 smithi099 ceph-mon[33865]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:54.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:53 smithi029 ceph-mon[30044]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:54.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:53 smithi029 ceph-mon[41899]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:55.878 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:55 smithi029 ceph-mon[30044]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:55.878 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:55 smithi029 ceph-mon[41899]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:56.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:55 smithi099 ceph-mon[33865]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:56.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:56 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:27:56.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:56 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:56.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:56 smithi029 ceph-mon[30044]: Deploying daemon osd.1 on smithi029 2024-03-29T04:27:56.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:56 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:27:56.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:56 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:56.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:56 smithi029 ceph-mon[41899]: Deploying daemon osd.1 on smithi029 2024-03-29T04:27:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:56 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:27:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:56 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:27:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:56 smithi099 ceph-mon[33865]: Deploying daemon osd.1 on smithi029 2024-03-29T04:27:57.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:57 smithi029 ceph-mon[30044]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:57.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:57 smithi029 ceph-mon[41899]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:58.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:57 smithi099 ceph-mon[33865]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:59.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:27:59 smithi029 ceph-mon[30044]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:27:59.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:27:59 smithi029 ceph-mon[41899]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:00.102 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:27:59 smithi099 ceph-mon[33865]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:02.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:01 smithi099 ceph-mon[33865]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:02.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:02.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:02.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:02.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[30044]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:02.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:02.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:02.115 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:02.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[41899]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:02.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:02.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:02.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:03.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:03 smithi029 ceph-mon[30044]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:03.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:03 smithi029 ceph-mon[41899]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:04.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:03 smithi099 ceph-mon[33865]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:04.355 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 1 on host 'smithi029' 2024-03-29T04:28:05.037 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[30044]: from='osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[30044]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[41899]: from='osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[41899]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:05.038 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:05.069 DEBUG:teuthology.orchestra.run.smithi029:osd.1> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.1.service 2024-03-29T04:28:05.072 INFO:tasks.cephadm:Deploying osd.2 on smithi029 with /dev/vg_nvme/lv_2... 2024-03-29T04:28:05.072 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-29T04:28:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:04 smithi099 ceph-mon[33865]: from='osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:28:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:04 smithi099 ceph-mon[33865]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:28:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:05.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:05.390 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:28:06.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: from='osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:06.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:06.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: from='osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:28:06.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: from='osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:06.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:06.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:06.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:06.170 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[52463]: 2024-03-29T04:28:05.811+0000 7fab32a6a700 -1 osd.1 0 waiting for initial osdmap 2024-03-29T04:28:06.170 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[52463]: 2024-03-29T04:28:05.814+0000 7fab2d865700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:28:06.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[30044]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:06.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[30044]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:28:06.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[41899]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[41899]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:06.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:06 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:07.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:06 smithi099 ceph-mon[33865]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:07.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:06 smithi099 ceph-mon[33865]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:28:07.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:07.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:07.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:06 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:07.483 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-29T04:28:07.484 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-29T04:28:07.484 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-03-29T04:28:07.484 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-03-29T04:28:07.484 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0096231 s, 1.1 GB/s 2024-03-29T04:28:07.484 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-03-29T04:28:07.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:28:07.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[30044]: pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[30044]: osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377] boot 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[30044]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[41899]: pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[41899]: osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377] boot 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[41899]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:28:07.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:07 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:08.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:07 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:28:08.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:07 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:28:08.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:07 smithi099 ceph-mon[33865]: pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:28:08.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:07 smithi099 ceph-mon[33865]: osd.1 [v2:172.21.15.29:6810/1074269377,v1:172.21.15.29:6811/1074269377] boot 2024-03-29T04:28:08.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:07 smithi099 ceph-mon[33865]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:28:08.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:07 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:28:08.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi029:vg_nvme/lv_2 2024-03-29T04:28:08.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:08 smithi029 ceph-mon[30044]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:28:08.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:08 smithi029 ceph-mon[41899]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:28:09.098 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:28:09.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:08 smithi099 ceph-mon[33865]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:28:09.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:09 smithi029 ceph-mon[30044]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:09.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:09 smithi029 ceph-mon[41899]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:10.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:09 smithi099 ceph-mon[33865]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:12.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:11 smithi099 ceph-mon[33865]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:12.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:28:12.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:28:12.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:12.121 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[30044]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:12.121 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:28:12.121 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:28:12.121 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:12.121 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[41899]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:12.122 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:28:12.122 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:28:12.122 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi029 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi029 to 8034M 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:13.011 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi029 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi029 to 8034M 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.012 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:13.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:13.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:13.013 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi029 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi029 to 8034M 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:13.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:13.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:14.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:14 smithi099 ceph-mon[33865]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:14.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:14 smithi029 ceph-mon[30044]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:14.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:14 smithi029 ceph-mon[41899]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:15.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:15 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1758852059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]: dispatch 2024-03-29T04:28:15.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:15 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]: dispatch 2024-03-29T04:28:15.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:15 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]': finished 2024-03-29T04:28:15.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:15 smithi099 ceph-mon[33865]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:28:15.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:15.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:15 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2539977098' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:28:15.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1758852059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]: dispatch 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]: dispatch 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]': finished 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[30044]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2539977098' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1758852059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]: dispatch 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]: dispatch 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9112d005-33dd-4646-acea-24c19aae33ab"}]': finished 2024-03-29T04:28:15.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[41899]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:28:15.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:15.420 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:15 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2539977098' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:28:16.350 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:16 smithi099 ceph-mon[33865]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:16.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:16 smithi029 ceph-mon[30044]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:16.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:16 smithi029 ceph-mon[41899]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:18.287 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:18 smithi029 ceph-mon[30044]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:18.287 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:18 smithi029 ceph-mon[41899]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:18.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:18 smithi099 ceph-mon[33865]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:20.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:20 smithi099 ceph-mon[33865]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:20.407 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:20 smithi029 ceph-mon[30044]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:20.407 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:20 smithi029 ceph-mon[41899]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:21.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:28:21.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:21.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:28:21.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:21.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:28:21.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:22.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:22 smithi029 ceph-mon[30044]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:22.303 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:22 smithi029 ceph-mon[30044]: Deploying daemon osd.2 on smithi029 2024-03-29T04:28:22.303 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:22 smithi029 ceph-mon[41899]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:22.303 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:22 smithi029 ceph-mon[41899]: Deploying daemon osd.2 on smithi029 2024-03-29T04:28:22.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:22 smithi099 ceph-mon[33865]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:22.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:22 smithi099 ceph-mon[33865]: Deploying daemon osd.2 on smithi029 2024-03-29T04:28:24.048 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:24 smithi029 ceph-mon[30044]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:24.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:24 smithi029 ceph-mon[41899]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:24.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:24 smithi099 ceph-mon[33865]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:26.260 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:26 smithi029 ceph-mon[30044]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:26.260 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:26 smithi029 ceph-mon[41899]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:26.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:26 smithi099 ceph-mon[33865]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:27.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:27.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:27.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:27.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[30044]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:27.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:27.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:27.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:27.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:27 smithi029 ceph-mon[41899]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:27.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:27 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:27.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:27 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:27.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:27 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:27.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:27 smithi099 ceph-mon[33865]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:29.857 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 2 on host 'smithi029' 2024-03-29T04:28:30.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:29 smithi099 ceph-mon[33865]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:29 smithi099 ceph-mon[33865]: from='osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:28:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:29 smithi099 ceph-mon[33865]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:28:30.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:29 smithi029 ceph-mon[41899]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:30.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:29 smithi029 ceph-mon[41899]: from='osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:28:30.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:29 smithi029 ceph-mon[41899]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:28:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:29 smithi029 ceph-mon[30044]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:29 smithi029 ceph-mon[30044]: from='osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:28:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:29 smithi029 ceph-mon[30044]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:28:30.478 DEBUG:teuthology.orchestra.run.smithi029:osd.2> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.2.service 2024-03-29T04:28:30.480 INFO:tasks.cephadm:Deploying osd.3 on smithi029 with /dev/vg_nvme/lv_1... 2024-03-29T04:28:30.480 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-29T04:28:30.806 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:28:31.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:28:31.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:28:31.045 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:28:31.046 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:31.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:31.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:31.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.047 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.047 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[58768]: 2024-03-29T04:28:30.792+0000 7f05228d7700 -1 osd.2 0 waiting for initial osdmap 2024-03-29T04:28:31.047 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[58768]: 2024-03-29T04:28:30.795+0000 7f051b6ce700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:28:31.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:28:31.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:30 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[30044]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[30044]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[30044]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[41899]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[41899]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[41899]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:28:31.955 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:31.956 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:31 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:32.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:31 smithi099 ceph-mon[33865]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:32.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:31 smithi099 ceph-mon[33865]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:32.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:31 smithi099 ceph-mon[33865]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:28:32.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:31 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:32.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:31 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:33.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:28:33.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:28:33.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[30044]: osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485] boot 2024-03-29T04:28:33.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[30044]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:28:33.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:33.162 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-29T04:28:33.162 INFO:teuthology.orchestra.run.smithi029.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-29T04:28:33.162 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 10+0 records in 2024-03-29T04:28:33.163 INFO:teuthology.orchestra.run.smithi029.stderr:10+0 records out 2024-03-29T04:28:33.163 INFO:teuthology.orchestra.run.smithi029.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00968957 s, 1.1 GB/s 2024-03-29T04:28:33.163 INFO:teuthology.orchestra.run.smithi029.stderr: stderr: 2024-03-29T04:28:33.163 INFO:teuthology.orchestra.run.smithi029.stderr:--> Zapping successful for: 2024-03-29T04:28:33.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:28:33.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:28:33.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[41899]: osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485] boot 2024-03-29T04:28:33.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[41899]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:28:33.163 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:33 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:33.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:33 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:28:33.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:33 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:28:33.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:33 smithi099 ceph-mon[33865]: osd.2 [v2:172.21.15.29:6818/2548163485,v1:172.21.15.29:6819/2548163485] boot 2024-03-29T04:28:33.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:33 smithi099 ceph-mon[33865]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:28:33.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:33 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:28:34.229 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[30044]: pgmap v77: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:34.229 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:28:34.229 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[30044]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[41899]: pgmap v77: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[41899]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:28:34.230 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:34 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:28:34.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi029:vg_nvme/lv_1 2024-03-29T04:28:34.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:34 smithi099 ceph-mon[33865]: pgmap v77: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:28:34.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:34 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:28:34.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:34 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:28:34.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:34 smithi099 ceph-mon[33865]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:28:34.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:34 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:28:34.854 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:28:35.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:28:35.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:35 smithi029 ceph-mon[30044]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:28:35.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:35 smithi029 ceph-mon[30044]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:28:35.097 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:28:35.097 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:35 smithi029 ceph-mon[41899]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:28:35.097 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:35 smithi029 ceph-mon[41899]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:28:35.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:28:35.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:35 smithi099 ceph-mon[33865]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:28:35.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:35 smithi099 ceph-mon[33865]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:28:35.945 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63169]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:28:35.946 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63169]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:35.946 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63169]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:35.946 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63169]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:36.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:36 smithi029 ceph-mon[30044]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:36.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:36 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:36.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:36 smithi029 ceph-mon[41899]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:36.223 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:36 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:36.223 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63186]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:28:36.223 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63186]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:36.224 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63186]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:36.224 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:28:35 smithi029 sudo[63186]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:36.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:36 smithi099 ceph-mon[33865]: pgmap v80: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:36.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:36 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:36.485 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63189]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:28:36.485 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63189]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:36.485 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63189]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:36.485 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63189]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:36.756 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63210]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:28:36.757 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63210]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:36.757 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63210]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:36.757 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63210]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:37.012 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63213]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:28:37.012 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63213]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:37.012 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63213]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:37.012 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:36 smithi029 sudo[63213]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:37.296 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63249]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:28:37.297 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63249]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:37.297 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63249]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:37.297 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63249]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.297 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.298 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:37.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:37.299 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:28:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:37.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63272]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:28:37.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63272]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:37.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63272]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:37.582 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63272]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:37.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63303]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:28:37.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63303]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:37.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63303]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:37.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:37 smithi029 sudo[63303]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:38.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 sudo[36686]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:28:38.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 sudo[36686]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:38.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 sudo[36686]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:38.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:37 smithi099 sudo[36686]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:38.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 sudo[36689]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi029 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi029 to 5356M 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:38.312 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:38.313 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi029 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi029 to 5356M 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:38.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:38.315 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:28:38.315 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:28:38.315 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:38.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 sudo[36689]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:38.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 sudo[36689]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 sudo[36689]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi029 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi029 to 5356M 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:38.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:38.350 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:38.350 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:28:38.350 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:28:38.350 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:38.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63498]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:28:38.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63498]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:38.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63498]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:38.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63498]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:38.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63501]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:28:38.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63501]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:28:38.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63501]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:28:38.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:38 smithi029 sudo[63501]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:28:39.255 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:39.255 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:28:39.255 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:39.255 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[41899]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:39.256 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:39.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:39.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:39.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:39.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:39.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:39.257 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:39 smithi029 ceph-mon[30044]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:39.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:39.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='client.24206 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi029:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:28:39.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:39.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:28:39.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:39 smithi099 ceph-mon[33865]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:40.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:40 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4111158631' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]: dispatch 2024-03-29T04:28:40.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:40 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]: dispatch 2024-03-29T04:28:40.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:40 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]': finished 2024-03-29T04:28:40.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:40 smithi099 ceph-mon[33865]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:28:40.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:40 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:40.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4111158631' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]: dispatch 2024-03-29T04:28:40.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]: dispatch 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]': finished 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[30044]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4111158631' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]: dispatch 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]: dispatch 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85f1c525-a42d-412d-843e-ef920c6ba6d2"}]': finished 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[41899]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:28:40.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:40 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:41.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:41 smithi099 ceph-mon[33865]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:41.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:41 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3404084821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:28:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:41 smithi029 ceph-mon[30044]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3404084821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:28:41.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:41 smithi029 ceph-mon[41899]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:41.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:41 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3404084821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:28:43.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:43 smithi029 ceph-mon[41899]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:43.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:43 smithi029 ceph-mon[30044]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:44.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:43 smithi099 ceph-mon[33865]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:45.870 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:45 smithi029 ceph-mon[30044]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:28:45.870 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:45 smithi029 ceph-mon[41899]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:28:46.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:45 smithi099 ceph-mon[33865]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:28:48.083 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:47 smithi029 ceph-mon[30044]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:28:48.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:28:48.084 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:48.084 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:47 smithi029 ceph-mon[41899]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:28:48.084 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:28:48.084 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:47 smithi099 ceph-mon[33865]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:28:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:28:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:28:48.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:48 smithi029 ceph-mon[30044]: Deploying daemon osd.3 on smithi029 2024-03-29T04:28:48.818 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:48 smithi029 ceph-mon[41899]: Deploying daemon osd.3 on smithi029 2024-03-29T04:28:49.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:48 smithi099 ceph-mon[33865]: Deploying daemon osd.3 on smithi029 2024-03-29T04:28:50.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:49 smithi099 ceph-mon[33865]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:28:50.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:49 smithi029 ceph-mon[30044]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:28:50.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:49 smithi029 ceph-mon[41899]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:28:51.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:51 smithi029 ceph-mon[30044]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.7 KiB/s rd, 76 KiB/s wr, 5 op/s 2024-03-29T04:28:51.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:51 smithi029 ceph-mon[41899]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.7 KiB/s rd, 76 KiB/s wr, 5 op/s 2024-03-29T04:28:52.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:51 smithi099 ceph-mon[33865]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.7 KiB/s rd, 76 KiB/s wr, 5 op/s 2024-03-29T04:28:53.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[41899]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:28:53.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:53.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:53.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:53.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[30044]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:28:53.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:53.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:53.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:53 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:54.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:53 smithi099 ceph-mon[33865]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:28:54.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:53 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:54.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:53 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:54.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:53 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:28:55.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:55 smithi029 ceph-mon[41899]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:28:55.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:55 smithi029 ceph-mon[30044]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:28:56.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:55 smithi099 ceph-mon[33865]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:28:56.433 INFO:teuthology.orchestra.run.smithi029.stdout:Created osd(s) 3 on host 'smithi029' 2024-03-29T04:28:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:56 smithi099 ceph-mon[33865]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:28:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:56 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:56 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:57.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:56 smithi029 ceph-mon[30044]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:28:57.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:56 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:57.132 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:56 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:57.132 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:56 smithi029 ceph-mon[41899]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:28:57.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:56 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:57.133 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:56 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:57.159 DEBUG:teuthology.orchestra.run.smithi029:osd.3> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.3.service 2024-03-29T04:28:57.162 INFO:tasks.cephadm:Deploying osd.4 on smithi099 with /dev/vg_nvme/lv_4... 2024-03-29T04:28:57.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-29T04:28:57.417 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:58.056 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:58.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:28:58.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:28:58.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:58.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:58.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:58.057 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:58.057 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[65409]: 2024-03-29T04:28:57.833+0000 7efd7c338700 -1 osd.3 0 waiting for initial osdmap 2024-03-29T04:28:58.057 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:28:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[65409]: 2024-03-29T04:28:57.837+0000 7efd77934700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:28:58.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:58.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:28:58.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:28:58.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:58.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:28:58.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:58.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:57 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:28:58.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[30044]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:58.811 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[30044]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:28:58.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:58 smithi099 ceph-mon[33865]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:58.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:58 smithi099 ceph-mon[33865]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:28:58.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:58 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:58.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:58 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:59.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:59.139 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:59.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[41899]: from='osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]': finished 2024-03-29T04:28:59.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[41899]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:28:59.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:59.139 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:58 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:59.366 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-29T04:28:59.366 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-29T04:28:59.366 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-29T04:28:59.366 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-29T04:28:59.366 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114603 s, 915 MB/s 2024-03-29T04:28:59.366 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-29T04:28:59.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:28:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:28:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:28:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:28:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271] boot 2024-03-29T04:28:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:28:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:28:59 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:29:00.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_4 2024-03-29T04:29:00.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:29:00.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:29:00.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:29:00.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:29:00.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271] boot 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: osd.3 [v2:172.21.15.29:6826/2031012271,v1:172.21.15.29:6827/2031012271] boot 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:29:00.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:28:59 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:29:00.262 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:29:00.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:29:00.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:29:00.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:00.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:01.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:29:01.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:01.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:01.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:01.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:01.904 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:01.904 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi029 2024-03-29T04:29:01.904 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi029 to 4017M 2024-03-29T04:29:01.904 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:01.904 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:01.904 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:01.905 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:01 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:02.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:02.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi029 2024-03-29T04:29:02.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi029 to 4017M 2024-03-29T04:29:02.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi029 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi029 to 4017M 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:02.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:01 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:03.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:03.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:03.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:03.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:03.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:03.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:03.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:03.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:03.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:04.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:03 smithi099 ceph-mon[33865]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:04.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:03 smithi099 ceph-mon[33865]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:04.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:03 smithi029 ceph-mon[30044]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:04.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:03 smithi029 ceph-mon[30044]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:04.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:03 smithi029 ceph-mon[41899]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:04.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:03 smithi029 ceph-mon[41899]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:04 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/391409087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]: dispatch 2024-03-29T04:29:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:04 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]: dispatch 2024-03-29T04:29:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:04 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]': finished 2024-03-29T04:29:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:04 smithi099 ceph-mon[33865]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:29:05.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:04 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:05.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/391409087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]: dispatch 2024-03-29T04:29:05.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]: dispatch 2024-03-29T04:29:05.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]': finished 2024-03-29T04:29:05.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[30044]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:29:05.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/391409087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]: dispatch 2024-03-29T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]: dispatch 2024-03-29T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8e08e1f-4fc9-43c2-bfda-aa4b4d006013"}]': finished 2024-03-29T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[41899]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:29:05.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:04 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:06.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:05 smithi099 ceph-mon[33865]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:06.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/4268194790' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:06.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:05 smithi029 ceph-mon[30044]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:06.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/4268194790' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:06.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:05 smithi029 ceph-mon[41899]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:06.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/4268194790' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:08.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:07 smithi099 ceph-mon[33865]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:08.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:07 smithi029 ceph-mon[30044]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:08.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:07 smithi029 ceph-mon[41899]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:10.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:09 smithi099 ceph-mon[33865]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:10.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:09 smithi029 ceph-mon[30044]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:10.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:09 smithi029 ceph-mon[41899]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:12.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:11 smithi099 ceph-mon[33865]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:12.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:29:12.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:12.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:11 smithi099 ceph-mon[33865]: Deploying daemon osd.4 on smithi099 2024-03-29T04:29:12.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[30044]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:12.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:29:12.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:12.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[30044]: Deploying daemon osd.4 on smithi099 2024-03-29T04:29:12.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[41899]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:12.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:29:12.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:12.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:11 smithi029 ceph-mon[41899]: Deploying daemon osd.4 on smithi099 2024-03-29T04:29:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:13 smithi099 ceph-mon[33865]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:14.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:13 smithi029 ceph-mon[30044]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:14.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:13 smithi029 ceph-mon[41899]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:15 smithi099 ceph-mon[33865]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:16.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:15 smithi029 ceph-mon[30044]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:16.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:15 smithi029 ceph-mon[41899]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:17.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:17.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:17.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:17.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:17 smithi099 ceph-mon[33865]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:17.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:17.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:17.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:17.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[41899]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:17.419 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:17 smithi029 ceph-mon[30044]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:19.486 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 4 on host 'smithi099' 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:19.793 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:29:19.794 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:19 smithi099 ceph-mon[33865]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:29:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[41899]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:29:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:19 smithi029 ceph-mon[30044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:29:20.102 DEBUG:teuthology.orchestra.run.smithi099:osd.4> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.4.service 2024-03-29T04:29:20.104 INFO:tasks.cephadm:Deploying osd.5 on smithi099 with /dev/vg_nvme/lv_3... 2024-03-29T04:29:20.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-29T04:29:20.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:29:20.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:29:20.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:20.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:20.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:20.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:20.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:20.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:20.599 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:29:20.487+0000 7fd2c8565700 -1 osd.4 0 waiting for initial osdmap 2024-03-29T04:29:20.599 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:29:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:29:20.491+0000 7fd2c1b5d700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:29:20.904 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:20.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:20.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:20 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:21.586 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:21 smithi099 ceph-mon[33865]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:29:21.586 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:21 smithi099 ceph-mon[33865]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:29:21.586 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:21.586 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:21 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:21.586 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:21 smithi099 ceph-mon[33865]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:21.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[30044]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:29:21.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[30044]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:29:21.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:21.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:21.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[30044]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:21.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[41899]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:29:21.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[41899]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:29:21.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:21.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:21.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:21 smithi029 ceph-mon[41899]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:29:22.692 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:22 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:29:22.692 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:22 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:29:22.692 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:22 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:22.692 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:22 smithi099 ceph-mon[33865]: osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052] boot 2024-03-29T04:29:22.692 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:22 smithi099 ceph-mon[33865]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:29:22.692 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:22 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:22.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:29:22.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:29:22.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:22.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[30044]: osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052] boot 2024-03-29T04:29:22.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[30044]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[41899]: osd.4 [v2:172.21.15.99:6800/4191064052,v1:172.21.15.99:6801/4191064052] boot 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[41899]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:29:22.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:22 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:29:23.242 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-29T04:29:23.242 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-29T04:29:23.242 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-29T04:29:23.243 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-29T04:29:23.243 INFO:teuthology.orchestra.run.smithi099.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124493 s, 842 MB/s 2024-03-29T04:29:23.243 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-29T04:29:23.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:23 smithi099 ceph-mon[33865]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:29:23.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:23 smithi099 ceph-mon[33865]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:23.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:23 smithi029 ceph-mon[30044]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:29:23.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:23 smithi029 ceph-mon[30044]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:23.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:23 smithi029 ceph-mon[41899]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:29:23.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:23 smithi029 ceph-mon[41899]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:24.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_3 2024-03-29T04:29:24.613 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:24 smithi099 ceph-mon[33865]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:29:24.773 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:29:24.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:24 smithi029 ceph-mon[30044]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:29:24.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:24 smithi029 ceph-mon[41899]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:29:25.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:29:25.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi099 2024-03-29T04:29:25.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi099 to 17088M 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: pgmap v119: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:25.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:25 smithi099 ceph-mon[33865]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:29:25.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:29:25.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi099 2024-03-29T04:29:25.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi099 to 17088M 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: pgmap v119: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[30044]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi099 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi099 to 17088M 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: pgmap v119: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:25.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:25 smithi029 ceph-mon[41899]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:29:26.947 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:26 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:26.947 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:26 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:26.947 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:26 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:27.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:26 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:27.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:26 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:27.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:26 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:27.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:26 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:27.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:26 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:27.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:26 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:28.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:27 smithi099 ceph-mon[33865]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:28.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:27 smithi099 ceph-mon[33865]: pgmap v121: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:28.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:27 smithi029 ceph-mon[30044]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:28.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:27 smithi029 ceph-mon[30044]: pgmap v121: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:28.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:27 smithi029 ceph-mon[41899]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:28.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:27 smithi029 ceph-mon[41899]: pgmap v121: 1 pgs: 1 remapped+peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:30.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:29:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/240739324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]: dispatch 2024-03-29T04:29:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]: dispatch 2024-03-29T04:29:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]': finished 2024-03-29T04:29:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:29:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:30.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:29 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/4284218877' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:30.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:29:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/240739324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]: dispatch 2024-03-29T04:29:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]: dispatch 2024-03-29T04:29:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]': finished 2024-03-29T04:29:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:29:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:30.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/4284218877' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:29:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/240739324' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]: dispatch 2024-03-29T04:29:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]: dispatch 2024-03-29T04:29:30.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1180a355-b228-4a88-9f25-23ed308ae14e"}]': finished 2024-03-29T04:29:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:29:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:30.171 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:29 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/4284218877' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:32.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:31 smithi099 ceph-mon[33865]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-29T04:29:32.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:31 smithi029 ceph-mon[30044]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-29T04:29:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:31 smithi029 ceph-mon[41899]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-03-29T04:29:33.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:33 smithi099 ceph-mon[33865]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:29:34.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:33 smithi029 ceph-mon[30044]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:29:34.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:33 smithi029 ceph-mon[41899]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:29:36.046 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:35 smithi099 ceph-mon[33865]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-29T04:29:36.047 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:29:36.047 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:36.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:35 smithi029 ceph-mon[30044]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-29T04:29:36.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:29:36.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:36.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:35 smithi029 ceph-mon[41899]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-29T04:29:36.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:29:36.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:37.040 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:36 smithi099 ceph-mon[33865]: Deploying daemon osd.5 on smithi099 2024-03-29T04:29:37.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:36 smithi029 ceph-mon[30044]: Deploying daemon osd.5 on smithi099 2024-03-29T04:29:37.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:36 smithi029 ceph-mon[41899]: Deploying daemon osd.5 on smithi099 2024-03-29T04:29:37.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:37 smithi099 ceph-mon[33865]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:29:38.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:37 smithi029 ceph-mon[30044]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:29:38.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:37 smithi029 ceph-mon[41899]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:29:39.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:39 smithi099 ceph-mon[33865]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:40.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:39 smithi029 ceph-mon[30044]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:40.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:39 smithi029 ceph-mon[41899]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:41.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:41 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:41.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:41 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:41.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:41 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:41.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:41 smithi099 ceph-mon[33865]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:41.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[30044]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:41.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:41.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:41.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:41.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:41 smithi029 ceph-mon[41899]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:43.705 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 5 on host 'smithi099' 2024-03-29T04:29:44.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:44.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: from='osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:29:44.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:29:44.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:43 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:44.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: from='osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:29:44.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:29:44.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:44.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: from='osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:29:44.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:29:44.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.170 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:43 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:44.515 DEBUG:teuthology.orchestra.run.smithi099:osd.5> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.5.service 2024-03-29T04:29:44.517 INFO:tasks.cephadm:Deploying osd.6 on smithi099 with /dev/vg_nvme/lv_2... 2024-03-29T04:29:44.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-29T04:29:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-mon[33865]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:29:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-mon[33865]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:29:45.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:45.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-mon[33865]: from='osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:45.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-mon[33865]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:45.078 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:29:44.806+0000 7fcf11110700 -1 osd.5 0 waiting for initial osdmap 2024-03-29T04:29:45.078 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:29:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:29:44.810+0000 7fcf0df0f700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[30044]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[30044]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[30044]: from='osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[30044]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[41899]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:29:45.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[41899]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:29:45.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:45.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[41899]: from='osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:45.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:44 smithi029 ceph-mon[41899]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:29:45.285 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:29:46.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:45 smithi099 ceph-mon[33865]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:46.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:45 smithi099 ceph-mon[33865]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:29:46.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:45 smithi099 ceph-mon[33865]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:29:46.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:46.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:45 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:46.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[30044]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:46.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[30044]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:29:46.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[30044]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:29:46.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:46.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:46.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[41899]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:29:46.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[41899]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:29:46.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[41899]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:29:46.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:46.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:45 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:47.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:46 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:29:47.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:46 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:29:47.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:46 smithi099 ceph-mon[33865]: osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202] boot 2024-03-29T04:29:47.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:46 smithi099 ceph-mon[33865]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:29:47.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:46 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:47.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:46 smithi099 ceph-mon[33865]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[30044]: osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202] boot 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[30044]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[30044]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:29:47.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:29:47.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:29:47.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[41899]: osd.5 [v2:172.21.15.99:6808/3786712202,v1:172.21.15.99:6809/3786712202] boot 2024-03-29T04:29:47.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[41899]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:29:47.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:29:47.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:46 smithi029 ceph-mon[41899]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:29:48.042 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-29T04:29:48.043 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-29T04:29:48.043 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-29T04:29:48.043 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-29T04:29:48.043 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0151065 s, 694 MB/s 2024-03-29T04:29:48.043 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-29T04:29:48.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:47 smithi099 ceph-mon[33865]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:48.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:47 smithi099 ceph-mon[33865]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:29:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:47 smithi029 ceph-mon[30044]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:47 smithi029 ceph-mon[30044]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:29:48.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:47 smithi029 ceph-mon[41899]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:48.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:47 smithi029 ceph-mon[41899]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:29:48.899 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:48 smithi099 ceph-mon[33865]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:29:48.901 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:48 smithi099 ceph-mon[33865]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:29:48.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_2 2024-03-29T04:29:49.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:48 smithi029 ceph-mon[30044]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:29:49.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:48 smithi029 ceph-mon[30044]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:29:49.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:48 smithi029 ceph-mon[41899]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:29:49.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:48 smithi029 ceph-mon[41899]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:29:49.509 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:29:49.947 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:49.947 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:49.947 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:49.948 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:49 smithi099 ceph-mon[33865]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:29:50.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:50.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[41899]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:29:50.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:29:50.170 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:49 smithi029 ceph-mon[30044]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:29:50.937 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:50 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi099 2024-03-29T04:29:50.937 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:50 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi099 to 8544M 2024-03-29T04:29:51.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:50 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi099 2024-03-29T04:29:51.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:50 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi099 to 8544M 2024-03-29T04:29:51.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:50 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi099 2024-03-29T04:29:51.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:50 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi099 to 8544M 2024-03-29T04:29:52.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:51 smithi099 ceph-mon[33865]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:52.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:51 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:52.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:51 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:52.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:51 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:52.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[41899]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:52.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:52.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:52.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:52.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[30044]: pgmap v141: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:52.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:29:52.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:29:52.169 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:51 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:29:53.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:52 smithi099 ceph-mon[33865]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:53.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:52 smithi029 ceph-mon[30044]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:53.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:52 smithi029 ceph-mon[41899]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:29:54.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:53 smithi029 ceph-mon[30044]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:54.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:53 smithi029 ceph-mon[30044]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-29T04:29:54.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:53 smithi029 ceph-mon[30044]: Cluster is now healthy 2024-03-29T04:29:54.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:53 smithi029 ceph-mon[41899]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:54.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:53 smithi029 ceph-mon[41899]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-29T04:29:54.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:53 smithi029 ceph-mon[41899]: Cluster is now healthy 2024-03-29T04:29:54.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:53 smithi099 ceph-mon[33865]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:29:54.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:53 smithi099 ceph-mon[33865]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-29T04:29:54.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:53 smithi099 ceph-mon[33865]: Cluster is now healthy 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/3400035516' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]: dispatch 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]: dispatch 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]': finished 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[30044]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/3405446159' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:55.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/3400035516' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]: dispatch 2024-03-29T04:29:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]: dispatch 2024-03-29T04:29:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]': finished 2024-03-29T04:29:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[41899]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:29:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:29:55.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:54 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/3405446159' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:55.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:54 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/3400035516' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]: dispatch 2024-03-29T04:29:55.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:54 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]: dispatch 2024-03-29T04:29:55.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:54 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55a1eb89-3491-442f-8139-4078c86d535b"}]': finished 2024-03-29T04:29:55.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:54 smithi099 ceph-mon[33865]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:29:55.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:54 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:29:55.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:54 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/3405446159' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:29:56.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:55 smithi029 ceph-mon[41899]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:29:56.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:55 smithi029 ceph-mon[30044]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:29:56.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:55 smithi099 ceph-mon[33865]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:29:58.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:57 smithi029 ceph-mon[41899]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:29:58.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:57 smithi029 ceph-mon[30044]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:29:58.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:57 smithi099 ceph-mon[33865]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:30:00.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:29:59 smithi029 ceph-mon[30044]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:30:00.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:29:59 smithi029 ceph-mon[41899]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:30:00.233 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:29:59 smithi099 ceph-mon[33865]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:30:00.961 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:00 smithi099 ceph-mon[33865]: overall HEALTH_OK 2024-03-29T04:30:00.962 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:30:00.962 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:00 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:01.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:00 smithi029 ceph-mon[30044]: overall HEALTH_OK 2024-03-29T04:30:01.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:30:01.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:00 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:01.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:00 smithi029 ceph-mon[41899]: overall HEALTH_OK 2024-03-29T04:30:01.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:30:01.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:00 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:02.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:01 smithi099 ceph-mon[33865]: Deploying daemon osd.6 on smithi099 2024-03-29T04:30:02.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:01 smithi099 ceph-mon[33865]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:30:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:01 smithi029 ceph-mon[30044]: Deploying daemon osd.6 on smithi099 2024-03-29T04:30:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:01 smithi029 ceph-mon[30044]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:30:02.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:01 smithi029 ceph-mon[41899]: Deploying daemon osd.6 on smithi099 2024-03-29T04:30:02.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:01 smithi029 ceph-mon[41899]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:30:04.233 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:03 smithi099 ceph-mon[33865]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:04.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:03 smithi029 ceph-mon[30044]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:04.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:03 smithi029 ceph-mon[41899]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:06.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:05 smithi099 ceph-mon[33865]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:06.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:06.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:05 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[30044]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[41899]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:06.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:05 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:08.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:07 smithi099 ceph-mon[33865]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:08.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:07 smithi029 ceph-mon[30044]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:08.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:07 smithi029 ceph-mon[41899]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:09.183 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 6 on host 'smithi099' 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:09.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:09.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:09.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:09 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:09.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:09 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:09.796 DEBUG:teuthology.orchestra.run.smithi099:osd.6> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.6.service 2024-03-29T04:30:09.800 INFO:tasks.cephadm:Deploying osd.7 on smithi099 with /dev/vg_nvme/lv_1... 2024-03-29T04:30:09.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-29T04:30:10.594 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:30:10.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-mon[33865]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:30:10.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-mon[33865]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:30:10.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:10.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-mon[33865]: from='osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:10.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-mon[33865]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:10.598 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:30:10.303+0000 7f6874e47700 -1 osd.6 0 waiting for initial osdmap 2024-03-29T04:30:10.598 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:30:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:30:10.310+0000 7f686dc3e700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:30:10.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[30044]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:30:10.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[30044]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:30:10.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:10.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[30044]: from='osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:10.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[30044]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:10.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[41899]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:30:10.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[41899]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:30:10.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:10.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[41899]: from='osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:10.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:10 smithi029 ceph-mon[41899]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:11.441 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:11 smithi099 ceph-mon[33865]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:30:11.441 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:11 smithi099 ceph-mon[33865]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:30:11.441 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:11.441 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:11 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:11.441 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:11 smithi099 ceph-mon[33865]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[30044]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:30:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[30044]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:30:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[30044]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:11.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[41899]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:30:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[41899]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:30:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:11 smithi029 ceph-mon[41899]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:30:12.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:12 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:30:12.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:12 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:30:12.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:12.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:12 smithi099 ceph-mon[33865]: osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739] boot 2024-03-29T04:30:12.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:12 smithi099 ceph-mon[33865]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:30:12.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:12 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[41899]: osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739] boot 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[41899]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:12.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:30:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:30:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[30044]: osd.6 [v2:172.21.15.99:6816/2650562739,v1:172.21.15.99:6817/2650562739] boot 2024-03-29T04:30:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[30044]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:30:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:12 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:30:13.450 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-29T04:30:13.450 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-29T04:30:13.451 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2024-03-29T04:30:13.451 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2024-03-29T04:30:13.451 INFO:teuthology.orchestra.run.smithi099.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0138271 s, 758 MB/s 2024-03-29T04:30:13.451 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2024-03-29T04:30:13.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:13 smithi099 ceph-mon[33865]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:30:13.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:13 smithi099 ceph-mon[33865]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:13.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:13 smithi099 ceph-mon[33865]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:30:13.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:13 smithi029 ceph-mon[41899]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:30:13.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:13 smithi029 ceph-mon[41899]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:13.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:13 smithi029 ceph-mon[41899]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:30:13.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:13 smithi029 ceph-mon[30044]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:30:13.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:13 smithi029 ceph-mon[30044]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:13.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:13 smithi029 ceph-mon[30044]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:30:14.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi099:vg_nvme/lv_1 2024-03-29T04:30:14.824 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi099 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi099 to 5696M 2024-03-29T04:30:15.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:15.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:15.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:15 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi099 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi099 to 5696M 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: pgmap v160: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi099 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:15.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi099 to 5696M 2024-03-29T04:30:15.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:15.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:15.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:15.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:15 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:16.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:16 smithi099 ceph-mon[33865]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:30:16.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:16 smithi029 ceph-mon[30044]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:30:16.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:16 smithi029 ceph-mon[41899]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:30:17.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:17 smithi099 ceph-mon[33865]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:17.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:30:17.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:30:17.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:17 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:17.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:17 smithi099 ceph-mon[33865]: pgmap v162: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:17.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[41899]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:17.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:30:17.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:30:17.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:17.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[41899]: pgmap v162: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:17.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[30044]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:17.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:30:17.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:30:17.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:17.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:17 smithi029 ceph-mon[30044]: pgmap v162: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:19 smithi099 ceph-mon[33865]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:30:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:19 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/824936707' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]: dispatch 2024-03-29T04:30:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:19 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]: dispatch 2024-03-29T04:30:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:19 smithi099 ceph-mon[33865]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]': finished 2024-03-29T04:30:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:19 smithi099 ceph-mon[33865]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:30:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:19 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[30044]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:30:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/824936707' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]: dispatch 2024-03-29T04:30:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]: dispatch 2024-03-29T04:30:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[30044]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]': finished 2024-03-29T04:30:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[30044]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:30:20.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:20.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[41899]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:30:20.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/824936707' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]: dispatch 2024-03-29T04:30:20.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]: dispatch 2024-03-29T04:30:20.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[41899]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f4c76c0-d2d8-42dd-9b72-89702f57b432"}]': finished 2024-03-29T04:30:20.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[41899]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:30:20.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:19 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:21.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:20 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/1343896534' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:30:21.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:20 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/1343896534' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:30:21.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:20 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/1343896534' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:30:22.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:21 smithi099 ceph-mon[33865]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-03-29T04:30:22.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:21 smithi029 ceph-mon[30044]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-03-29T04:30:22.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:21 smithi029 ceph-mon[41899]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-03-29T04:30:24.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:23 smithi099 ceph-mon[33865]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:30:24.167 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:23 smithi029 ceph-mon[30044]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:30:24.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:23 smithi029 ceph-mon[41899]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:30:26.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:25 smithi099 ceph-mon[33865]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-29T04:30:26.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:25 smithi029 ceph-mon[30044]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-29T04:30:26.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:25 smithi029 ceph-mon[41899]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-29T04:30:27.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:26 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:30:27.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:26 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:27.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:26 smithi099 ceph-mon[33865]: Deploying daemon osd.7 on smithi099 2024-03-29T04:30:27.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:26 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:30:27.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:26 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:27.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:26 smithi029 ceph-mon[30044]: Deploying daemon osd.7 on smithi099 2024-03-29T04:30:27.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:26 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:30:27.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:26 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:27.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:26 smithi029 ceph-mon[41899]: Deploying daemon osd.7 on smithi099 2024-03-29T04:30:28.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:27 smithi099 ceph-mon[33865]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:30:28.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:27 smithi029 ceph-mon[30044]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:30:28.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:27 smithi029 ceph-mon[41899]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:30:29.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:29 smithi099 ceph-mon[33865]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:30.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:29 smithi029 ceph-mon[30044]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:30.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:29 smithi029 ceph-mon[41899]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:32.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:31 smithi099 ceph-mon[33865]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:32.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:31 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:32.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:31 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:32.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:31 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[30044]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[41899]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:32.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:32.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:31 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:33.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:33 smithi099 ceph-mon[33865]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:34.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:33 smithi029 ceph-mon[30044]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:34.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:33 smithi029 ceph-mon[41899]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:34.947 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 7 on host 'smithi099' 2024-03-29T04:30:35.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:35.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:35 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:35.599 DEBUG:teuthology.orchestra.run.smithi099:osd.7> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.7.service 2024-03-29T04:30:35.605 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-29T04:30:35.605 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd stat -f json 2024-03-29T04:30:35.636 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:30:35.637 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:35.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.638 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:35.638 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:35.639 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:30:35.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:30:35.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:35.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:35.640 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:35 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:35.878 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[41899]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[41899]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[41899]: from='osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[41899]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[30044]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[30044]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[30044]: from='osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:36.340 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:36 smithi029 ceph-mon[30044]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:36.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-mon[33865]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:30:36.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-mon[33865]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:30:36.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:36.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-mon[33865]: from='osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:36.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-mon[33865]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2024-03-29T04:30:36.498 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:30:36.225+0000 7fa80870a700 -1 osd.7 0 waiting for initial osdmap 2024-03-29T04:30:36.498 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:30:36 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:30:36.235+0000 7fa802d04700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:30:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:37 smithi099 ceph-mon[33865]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:30:37.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:37 smithi099 ceph-mon[33865]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:30:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:37 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:37.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:37 smithi099 ceph-mon[33865]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[30044]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[30044]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[30044]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[41899]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2024-03-29T04:30:37.522 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[41899]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:30:37.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:37.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:37.523 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:37 smithi029 ceph-mon[41899]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:30:37.999 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: purged_snaps scrub starts 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: purged_snaps scrub ok 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879] boot 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:38.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:38 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/905659274' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:30:38.667 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: purged_snaps scrub starts 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: purged_snaps scrub ok 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879] boot 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/905659274' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: purged_snaps scrub starts 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: purged_snaps scrub ok 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:38.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: osd.7 [v2:172.21.15.99:6824/2612020879,v1:172.21.15.99:6825/2612020879] boot 2024-03-29T04:30:38.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:30:38.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:30:38.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:38 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/905659274' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:30:38.742 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1711686637,"num_in_osds":8,"osd_in_since":1711686619,"num_remapped_pgs":1} 2024-03-29T04:30:38.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:30:39.041 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:30:39.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:30:39.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: Detected new or changed devices on smithi099 2024-03-29T04:30:39.413 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi099 to 4272M 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.414 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[30044]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: Detected new or changed devices on smithi099 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi099 to 4272M 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.415 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:39 smithi029 ceph-mon[41899]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:39.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:30:39.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: Detected new or changed devices on smithi099 2024-03-29T04:30:39.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi099 to 4272M 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:39.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:39 smithi099 ceph-mon[33865]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:40.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:40 smithi029 ceph-mon[30044]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:30:40.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:40 smithi029 ceph-mon[30044]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:30:40.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:40 smithi029 ceph-mon[41899]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:30:40.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:40 smithi029 ceph-mon[41899]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:30:40.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:40 smithi099 ceph-mon[33865]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:30:40.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:40 smithi099 ceph-mon[33865]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:30:41.183 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:30:41.183 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":59,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:30:40.524080+0000","last_up_change":"2024-03-29T04:30:37.220270+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6803","nonce":517562633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6805","nonce":517562633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6809","nonce":517562633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6807","nonce":517562633}]},"public_addr":"172.21.15.29:6803/517562633","cluster_addr":"172.21.15.29:6805/517562633","heartbeat_back_addr":"172.21.15.29:6809/517562633","heartbeat_front_addr":"172.21.15.29:6807/517562633","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6811","nonce":1074269377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6813","nonce":1074269377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6817","nonce":1074269377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6815","nonce":1074269377}]},"public_addr":"172.21.15.29:6811/1074269377","cluster_addr":"172.21.15.29:6813/1074269377","heartbeat_back_addr":"172.21.15.29:6817/1074269377","heartbeat_front_addr":"172.21.15.29:6815/1074269377","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6819","nonce":2548163485}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6821","nonce":2548163485}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6825","nonce":2548163485}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6823","nonce":2548163485}]},"public_addr":"172.21.15.29:6819/2548163485","cluster_addr":"172.21.15.29:6821/2548163485","heartbeat_back_addr":"172.21.15.29:6825/2548163485","heartbeat_front_addr":"172.21.15.29:6823/2548163485","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6827","nonce":2031012271}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6829","nonce":2031012271}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6833","nonce":2031012271}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6831","nonce":2031012271}]},"public_addr":"172.21.15.29:6827/2031012271","cluster_addr":"172.21.15.29:6829/2031012271","heartbeat_back_addr":"172.21.15.29:6833/2031012271","heartbeat_front_addr":"172.21.15.29:6831/2031012271","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:30:41.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:41 smithi099 ceph-mon[33865]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:30:41.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:41 smithi099 ceph-mon[33865]: pgmap v182: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:41.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:41 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3067472535' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:30:41.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:41 smithi029 ceph-mon[30044]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:30:41.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:41 smithi029 ceph-mon[30044]: pgmap v182: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:41.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3067472535' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:30:41.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:41 smithi029 ceph-mon[41899]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:30:41.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:41 smithi029 ceph-mon[41899]: pgmap v182: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:41.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:41 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3067472535' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:30:41.941 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-29T04:28:33.017273+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-29T04:30:41.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-29T04:30:42.204 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:30:44.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:43 smithi099 ceph-mon[33865]: pgmap v183: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:44.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:43 smithi029 ceph-mon[30044]: pgmap v183: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:44.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:43 smithi029 ceph-mon[41899]: pgmap v183: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:44.228 INFO:teuthology.orchestra.run.smithi029.stdout:pg_num: 1 2024-03-29T04:30:44.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:44 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2380127007' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:30:44.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:44 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2380127007' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:30:45.064 INFO:tasks.cephadm:Adding prometheus.a on smithi099 2024-03-29T04:30:45.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch apply prometheus '1;smithi099=a' 2024-03-29T04:30:45.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:44 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2380127007' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:30:45.326 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:30:46.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:45 smithi099 ceph-mon[33865]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:30:46.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:45 smithi029 ceph-mon[30044]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:30:46.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:45 smithi029 ceph-mon[41899]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:30:47.218 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled prometheus update... 2024-03-29T04:30:47.851 DEBUG:teuthology.orchestra.run.smithi099:prometheus.a> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service 2024-03-29T04:30:47.895 INFO:tasks.cephadm:Adding node-exporter.a on smithi029 2024-03-29T04:30:47.896 INFO:tasks.cephadm:Adding node-exporter.b on smithi099 2024-03-29T04:30:47.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch apply node-exporter '2;smithi029=a;smithi099=b' 2024-03-29T04:30:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:47 smithi099 ceph-mon[33865]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-03-29T04:30:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:48.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:47 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[30044]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:48.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[41899]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-03-29T04:30:48.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:48.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:30:48.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:30:48.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:30:48.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:47 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:48.219 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:30:49.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:48 smithi099 ceph-mon[33865]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:49.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:48 smithi099 ceph-mon[33865]: Saving service prometheus spec with placement smithi099=a;count:1 2024-03-29T04:30:49.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:48 smithi099 ceph-mon[33865]: Deploying daemon prometheus.a on smithi099 2024-03-29T04:30:49.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:48 smithi029 ceph-mon[30044]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:49.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:48 smithi029 ceph-mon[30044]: Saving service prometheus spec with placement smithi099=a;count:1 2024-03-29T04:30:49.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:48 smithi029 ceph-mon[30044]: Deploying daemon prometheus.a on smithi099 2024-03-29T04:30:49.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:48 smithi029 ceph-mon[41899]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:49.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:48 smithi029 ceph-mon[41899]: Saving service prometheus spec with placement smithi099=a;count:1 2024-03-29T04:30:49.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:48 smithi029 ceph-mon[41899]: Deploying daemon prometheus.a on smithi099 2024-03-29T04:30:50.069 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:49 smithi029 ceph-mon[30044]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-29T04:30:50.069 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:49 smithi029 ceph-mon[41899]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-29T04:30:50.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:49 smithi099 ceph-mon[33865]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-03-29T04:30:50.810 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled node-exporter update... 2024-03-29T04:30:51.874 DEBUG:teuthology.orchestra.run.smithi029:node-exporter.a> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service 2024-03-29T04:30:51.877 DEBUG:teuthology.orchestra.run.smithi099:node-exporter.b> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service 2024-03-29T04:30:51.880 INFO:tasks.cephadm:Adding alertmanager.a on smithi029 2024-03-29T04:30:51.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch apply alertmanager '1;smithi029=a' 2024-03-29T04:30:52.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:51 smithi099 ceph-mon[33865]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-03-29T04:30:52.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:51 smithi099 ceph-mon[33865]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi099=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:52.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:51 smithi099 ceph-mon[33865]: Saving service node-exporter spec with placement smithi029=a;smithi099=b;count:2 2024-03-29T04:30:52.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:51 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:52.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[30044]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-03-29T04:30:52.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[30044]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi099=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:52.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[30044]: Saving service node-exporter spec with placement smithi029=a;smithi099=b;count:2 2024-03-29T04:30:52.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:52.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[41899]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-03-29T04:30:52.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[41899]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi029=a;smithi099=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:52.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[41899]: Saving service node-exporter spec with placement smithi029=a;smithi099=b;count:2 2024-03-29T04:30:52.169 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:51 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:52.628 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:30:54.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:53 smithi099 ceph-mon[33865]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:30:54.167 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:53 smithi029 ceph-mon[30044]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:30:54.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:53 smithi029 ceph-mon[41899]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:30:56.019 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled alertmanager update... 2024-03-29T04:30:56.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:55 smithi099 ceph-mon[33865]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:30:56.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:55 smithi029 ceph-mon[30044]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:30:56.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:55 smithi029 ceph-mon[41899]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:30:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:57 smithi099 ceph-mon[33865]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:57 smithi099 ceph-mon[33865]: Saving service alertmanager spec with placement smithi029=a;count:1 2024-03-29T04:30:57.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:57 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:57.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:57 smithi029 ceph-mon[30044]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:57.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:57 smithi029 ceph-mon[30044]: Saving service alertmanager spec with placement smithi029=a;count:1 2024-03-29T04:30:57.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:57 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:57.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:57 smithi029 ceph-mon[41899]: from='client.24355 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi029=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:30:57.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:57 smithi029 ceph-mon[41899]: Saving service alertmanager spec with placement smithi029=a;count:1 2024-03-29T04:30:57.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:57 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:30:57.571 DEBUG:teuthology.orchestra.run.smithi029:alertmanager.a> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@alertmanager.a.service 2024-03-29T04:30:57.574 INFO:tasks.cephadm:Adding grafana.a on smithi099 2024-03-29T04:30:57.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph orch apply grafana '1;smithi099=a' 2024-03-29T04:30:58.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:30:58 smithi099 ceph-mon[33865]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:58.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:30:58 smithi029 ceph-mon[30044]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:58.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:30:58 smithi029 ceph-mon[41899]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:30:59.483 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:31:00.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:00 smithi099 ceph-mon[33865]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:00.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:00 smithi029 ceph-mon[30044]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:00.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:00 smithi029 ceph-mon[41899]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:01.974 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled grafana update... 2024-03-29T04:31:02.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mon[33865]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:02.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:31:02.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[30044]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[41899]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:02.418 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:31:02.419 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' 2024-03-29T04:31:02.873 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ignoring --setuser ceph since I am not root 2024-03-29T04:31:02.873 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ignoring --setgroup ceph since I am not root 2024-03-29T04:31:02.873 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:02 smithi099 ceph-mgr[36007]: -- 172.21.15.99:0/585605617 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5565380069c0 con 0x556537fdcc00 2024-03-29T04:31:02.874 DEBUG:teuthology.orchestra.run.smithi099:grafana.a> sudo journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service 2024-03-29T04:31:02.877 INFO:tasks.cephadm:Setting up client nodes... 2024-03-29T04:31:02.877 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-29T04:31:02.902 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setuser ceph since I am not root 2024-03-29T04:31:02.903 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setgroup ceph since I am not root 2024-03-29T04:31:02.903 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:02 smithi029 ceph-mgr[30326]: -- 172.21.15.29:0/3063556106 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5580809329c0 con 0x558080908800 2024-03-29T04:31:03.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-mon[33865]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:31:03.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-mon[33865]: Saving service grafana spec with placement smithi099=a;count:1 2024-03-29T04:31:03.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-mon[33865]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:31:03.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-mon[33865]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:31:03.150 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[30044]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[30044]: Saving service grafana spec with placement smithi099=a;count:1 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[30044]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[30044]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[41899]: from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[41899]: Saving service grafana spec with placement smithi099=a;count:1 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[41899]: from='mgr.14152 172.21.15.29:0/3796283426' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:31:03.160 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-mon[41899]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:31:03.417 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:03.183+0000 7fd7c60991c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:31:03.417 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:03.280+0000 7fd7c60991c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:31:03.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:03.152+0000 7f2182dd21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:31:03.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:03.242+0000 7f2182dd21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:31:04.098 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:03.863+0000 7f2182dd21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:31:04.098 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:03.951+0000 7f2182dd21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:31:04.098 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.034+0000 7f2182dd21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:31:04.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:03 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:03.939+0000 7fd7c60991c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:31:04.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.031+0000 7fd7c60991c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:31:04.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.118+0000 7fd7c60991c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:31:04.510 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.270+0000 7fd7c60991c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:31:04.510 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.353+0000 7fd7c60991c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:31:04.510 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.533+0000 7fd7c60991c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:31:04.585 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.182+0000 7f2182dd21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:31:04.585 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.261+0000 7f2182dd21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:31:04.585 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.430+0000 7f2182dd21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:31:04.585 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.608+0000 7f2182dd21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:31:04.773 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.718+0000 7fd7c60991c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:31:04.847 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.692+0000 7f2182dd21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:31:04.847 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.785+0000 7f2182dd21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:31:05.098 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:04.870+0000 7f2182dd21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:31:05.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.807+0000 7fd7c60991c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:31:05.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.904+0000 7fd7c60991c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:31:05.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:04.995+0000 7fd7c60991c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:31:05.312 INFO:teuthology.orchestra.run.smithi029.stdout:[client.0] 2024-03-29T04:31:05.313 INFO:teuthology.orchestra.run.smithi029.stdout: key = AQAJRAZmYvrlExAAIPWSGefnGlQW9Sw2aVMgWg== 2024-03-29T04:31:05.622 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1531098994' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:05.622 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1531098994' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:31:05.623 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1531098994' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:05.623 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1531098994' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:31:05.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1531098994' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:05.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1531098994' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:31:05.918 DEBUG:teuthology.orchestra.run.smithi029:> set -ex 2024-03-29T04:31:05.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-29T04:31:05.918 DEBUG:teuthology.orchestra.run.smithi029:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-29T04:31:06.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-29T04:31:06.167 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:06 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:06.093+0000 7fd7c60991c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:31:06.213 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:05.905+0000 7f2182dd21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:31:06.213 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:06.033+0000 7f2182dd21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:31:06.213 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:06.237+0000 7f2182dd21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:31:06.262 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:31:06.505 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:06.477+0000 7f2182dd21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:31:06.630 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:06 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:06.226+0000 7fd7c60991c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:31:06.631 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:06 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:06.425+0000 7fd7c60991c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:31:06.770 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:06.659+0000 7f2182dd21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:31:06.917 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:06 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:06.655+0000 7fd7c60991c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:31:06.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:06 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:06.835+0000 7fd7c60991c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:31:07.083 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:06.845+0000 7f2182dd21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:31:07.265 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:07 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:07.021+0000 7fd7c60991c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:31:07.348 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:07.107+0000 7f2182dd21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:31:07.348 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:07.193+0000 7f2182dd21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:31:07.667 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:07 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:07.289+0000 7fd7c60991c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:31:07.667 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:07 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:07.378+0000 7fd7c60991c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:31:07.848 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:07.583+0000 7f2182dd21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:31:08.168 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:07 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:07.773+0000 7fd7c60991c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:31:08.597 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:08.357+0000 7f2182dd21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:31:08.864 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:08.581+0000 7fd7c60991c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:31:09.067 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:08.658+0000 7f2182dd21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:31:09.067 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:08.972+0000 7f2182dd21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:31:09.167 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:08.888+0000 7fd7c60991c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:31:09.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-mon[33865]: Standby manager daemon x restarted 2024-03-29T04:31:09.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-mon[33865]: Standby manager daemon x started 2024-03-29T04:31:09.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:31:09.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:31:09.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:31:09.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:09.168+0000 7f2182dd21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: [29/Mar/2024:04:31:09] ENGINE Bus STARTING 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: CherryPy Checker: 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: The Application mounted at '' has an empty config. 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: [29/Mar/2024:04:31:09] ENGINE Serving on http://:::9283 2024-03-29T04:31:09.349 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: [29/Mar/2024:04:31:09] ENGINE Bus STARTED 2024-03-29T04:31:09.444 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:09.206+0000 7fd7c60991c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:31:09.444 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:09.410+0000 7fd7c60991c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:31:09.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[30044]: Standby manager daemon x restarted 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[30044]: Standby manager daemon x started 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[41899]: Standby manager daemon x restarted 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[41899]: Standby manager daemon x started 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:31:09.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/2339977724' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:31:09.679 INFO:teuthology.orchestra.run.smithi099.stdout:[client.1] 2024-03-29T04:31:09.679 INFO:teuthology.orchestra.run.smithi099.stdout: key = AQANRAZmojKwKRAAeGneTo2RFuAjtFBJt9F0aQ== 2024-03-29T04:31:09.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:09] ENGINE Bus STARTING 2024-03-29T04:31:09.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: CherryPy Checker: 2024-03-29T04:31:09.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: The Application mounted at '' has an empty config. 2024-03-29T04:31:09.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:31:09.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:09] ENGINE Serving on http://:::9283 2024-03-29T04:31:09.918 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:09 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:09] ENGINE Bus STARTED 2024-03-29T04:31:10.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: mgrmap e17: y(active, since 4m), standbys: x 2024-03-29T04:31:10.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: Active manager daemon y restarted 2024-03-29T04:31:10.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: Activating manager daemon y 2024-03-29T04:31:10.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:31:10.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: mgrmap e18: y(active, starting, since 0.0060548s), standbys: x 2024-03-29T04:31:10.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:31:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: Manager daemon y is now available 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='client.? 172.21.15.99:0/1772886403' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:10.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:10 smithi099 ceph-mon[33865]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:31:10.347 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2024-03-29T04:31:10.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-03-29T04:31:10.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-29T04:31:10.392 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-03-29T04:31:10.393 DEBUG:teuthology.orchestra.run.smithi029:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-29T04:31:10.544 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: mgrmap e17: y(active, since 4m), standbys: x 2024-03-29T04:31:10.544 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: Active manager daemon y restarted 2024-03-29T04:31:10.544 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: Activating manager daemon y 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: mgrmap e18: y(active, starting, since 0.0060548s), standbys: x 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:31:10.545 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: Manager daemon y is now available 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:31:10.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='client.? 172.21.15.99:0/1772886403' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[30044]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: mgrmap e17: y(active, since 4m), standbys: x 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: Active manager daemon y restarted 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: Activating manager daemon y 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: mgrmap e18: y(active, starting, since 0.0060548s), standbys: x 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:31:10.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: Manager daemon y is now available 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:31:10.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='client.? 172.21.15.99:0/1772886403' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:31:10.549 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:10 smithi029 ceph-mon[41899]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:31:10.680 INFO:teuthology.misc.health.smithi029.stdout: 2024-03-29T04:31:10.680 INFO:teuthology.misc.health.smithi029.stdout:{"epoch":60,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:31:09.412753+0000","last_up_change":"2024-03-29T04:30:37.220270+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6803","nonce":517562633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6805","nonce":517562633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6809","nonce":517562633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6807","nonce":517562633}]},"public_addr":"172.21.15.29:6803/517562633","cluster_addr":"172.21.15.29:6805/517562633","heartbeat_back_addr":"172.21.15.29:6809/517562633","heartbeat_front_addr":"172.21.15.29:6807/517562633","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6811","nonce":1074269377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6813","nonce":1074269377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6817","nonce":1074269377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6815","nonce":1074269377}]},"public_addr":"172.21.15.29:6811/1074269377","cluster_addr":"172.21.15.29:6813/1074269377","heartbeat_back_addr":"172.21.15.29:6817/1074269377","heartbeat_front_addr":"172.21.15.29:6815/1074269377","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6819","nonce":2548163485}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6821","nonce":2548163485}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6825","nonce":2548163485}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6823","nonce":2548163485}]},"public_addr":"172.21.15.29:6819/2548163485","cluster_addr":"172.21.15.29:6821/2548163485","heartbeat_back_addr":"172.21.15.29:6825/2548163485","heartbeat_front_addr":"172.21.15.29:6823/2548163485","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6827","nonce":2031012271}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6829","nonce":2031012271}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6833","nonce":2031012271}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6831","nonce":2031012271}]},"public_addr":"172.21.15.29:6827/2031012271","cluster_addr":"172.21.15.29:6829/2031012271","heartbeat_back_addr":"172.21.15.29:6833/2031012271","heartbeat_front_addr":"172.21.15.29:6831/2031012271","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:31:10.691 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-29T04:31:10.691 INFO:tasks.cephadm:Creating RBD pool 2024-03-29T04:31:10.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-03-29T04:31:10.962 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: [29/Mar/2024:04:31:10] ENGINE Bus STARTING 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: [29/Mar/2024:04:31:10] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: [29/Mar/2024:04:31:10] ENGINE Bus STARTED 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: mgrmap e19: y(active, since 1.01051s), standbys: x 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3516002285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: [29/Mar/2024:04:31:10] ENGINE Bus STARTING 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: [29/Mar/2024:04:31:10] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:31:11.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: [29/Mar/2024:04:31:10] ENGINE Bus STARTED 2024-03-29T04:31:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: mgrmap e19: y(active, since 1.01051s), standbys: x 2024-03-29T04:31:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:11 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3516002285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: [29/Mar/2024:04:31:10] ENGINE Bus STARTING 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: [29/Mar/2024:04:31:10] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: [29/Mar/2024:04:31:10] ENGINE Bus STARTED 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: mgrmap e19: y(active, since 1.01051s), standbys: x 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:11.689 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:11 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3516002285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[30044]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[30044]: mgrmap e20: y(active, since 2s), standbys: x 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[41899]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[41899]: mgrmap e20: y(active, since 2s), standbys: x 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:12.516 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:12 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:12.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:12 smithi099 ceph-mon[33865]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:12.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:12 smithi099 ceph-mon[33865]: mgrmap e20: y(active, since 2s), standbys: x 2024-03-29T04:31:12.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:12 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:12.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:12 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.898 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'rbd' created 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4079200898' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4079200898' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:13.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:31:13.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:13 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4079200898' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:31:14.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:13 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:31:14.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:31:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:31:14.670 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi099 to 4016M 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4079200898' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: mgrmap e21: y(active, since 4s), standbys: x 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi099 to 4016M 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4079200898' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:31:14.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: mgrmap e21: y(active, since 4s), standbys: x 2024-03-29T04:31:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:14 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:14.953 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:15.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi099 to 4016M 2024-03-29T04:31:15.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:31:15.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:31:15.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:15.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:31:15.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4079200898' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: mgrmap e21: y(active, since 4s), standbys: x 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:15.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:14 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[30044]: Deploying daemon alertmanager.a on smithi029 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[30044]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:31:15.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:31:15.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:31:15.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:31:15.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:31:15.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[41899]: Deploying daemon alertmanager.a on smithi029 2024-03-29T04:31:15.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:15 smithi029 ceph-mon[41899]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:31:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:15 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:31:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:15 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:31:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:15 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:31:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:15 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:31:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:15 smithi099 ceph-mon[33865]: Deploying daemon alertmanager.a on smithi029 2024-03-29T04:31:16.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:15 smithi099 ceph-mon[33865]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:31:16.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:16 smithi029 ceph-mon[41899]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:16.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:16 smithi029 ceph-mon[41899]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:31:16.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:16 smithi029 ceph-mon[41899]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:31:16.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:16 smithi029 ceph-mon[30044]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:16.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:16 smithi029 ceph-mon[30044]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:31:16.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:16 smithi029 ceph-mon[30044]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:31:17.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:16 smithi099 ceph-mon[33865]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:17.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:16 smithi099 ceph-mon[33865]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:31:17.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:16 smithi099 ceph-mon[33865]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:31:18.635 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rbd' on pool 'rbd' 2024-03-29T04:31:18.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:18 smithi029 ceph-mon[30044]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-03-29T04:31:18.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:18 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/648727759' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:31:18.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:18 smithi029 ceph-mon[41899]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-03-29T04:31:18.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:18 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/648727759' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:31:19.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:18 smithi099 ceph-mon[33865]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-03-29T04:31:19.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:18 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/648727759' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:31:19.917 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:19 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/648727759' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:31:19.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:19 smithi029 ceph-mon[41899]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:31:19.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:19 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/648727759' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:31:19.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:19 smithi029 ceph-mon[30044]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:31:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:19 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/648727759' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:31:20.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:19 smithi099 ceph-mon[33865]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:31:20.123 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:31:20.124 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:31:20.124 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:31:20.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:20 smithi029 ceph-mon[30044]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 21 op/s 2024-03-29T04:31:20.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:20 smithi029 ceph-mon[30044]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:31:20.635 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:20 smithi029 ceph-mon[30044]: Cluster is now healthy 2024-03-29T04:31:20.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:20 smithi029 ceph-mon[41899]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 21 op/s 2024-03-29T04:31:20.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:20 smithi029 ceph-mon[41899]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:31:20.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:20 smithi029 ceph-mon[41899]: Cluster is now healthy 2024-03-29T04:31:21.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:20 smithi099 ceph-mon[33865]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 21 op/s 2024-03-29T04:31:21.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:20 smithi099 ceph-mon[33865]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:31:21.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:20 smithi099 ceph-mon[33865]: Cluster is now healthy 2024-03-29T04:31:21.524 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:22.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:22 smithi029 ceph-mon[41899]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-03-29T04:31:22.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:22 smithi029 ceph-mon[30044]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-03-29T04:31:23.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:22 smithi099 ceph-mon[33865]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 0 B/s wr, 17 op/s 2024-03-29T04:31:24.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:31:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:31:24.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:24 smithi029 ceph-mon[30044]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:31:24.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:24 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:24.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:24 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:24.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:24 smithi029 ceph-mon[41899]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:31:24.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:24 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:24.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:24 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:24.919 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:24 smithi029 systemd[1]: Starting Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:31:25.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:24 smithi099 ceph-mon[33865]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:31:25.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:24 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:25.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:24 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:25.668 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:25 smithi029 podman[72943]: 2024-03-29 04:31:25.405295663 +0000 UTC m=+0.020627694 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:31:25.668 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:25 smithi029 podman[72943]: 2024-03-29 04:31:25.514651372 +0000 UTC m=+0.129983395 volume create 3382c56fbb7de184134671a14078be08cb24717e0e7e51dc926a2ea2252f1cd9 2024-03-29T04:31:25.668 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:25 smithi029 podman[72943]: 2024-03-29 04:31:25.607004434 +0000 UTC m=+0.222336519 container create 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:31:26.168 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 podman[72943]: 2024-03-29 04:31:26.131945874 +0000 UTC m=+0.747277897 container init 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:31:26.168 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 podman[72943]: 2024-03-29 04:31:26.137256126 +0000 UTC m=+0.752588151 container start 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:31:26.168 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.149Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:31:26.168 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.149Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:31:26.168 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.150Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2024-03-29T04:31:26.169 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.151Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:31:26.169 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.179Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:31:26.169 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.180Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:31:26.169 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.182Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:31:26.169 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:26.182Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:31:26.290 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:31:26.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-mon[30044]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:26.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2479641963' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:31:26.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-mon[41899]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:26.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2479641963' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:31:26.918 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 bash[72943]: 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 2024-03-29T04:31:26.918 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:26 smithi029 systemd[1]: Started Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:31:27.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:26 smithi099 ceph-mon[33865]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:27.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:26 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2479641963' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:31:27.281 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:26] ENGINE Bus STOPPING 2024-03-29T04:31:27.281 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:31:27.281 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:26] ENGINE Bus STOPPED 2024-03-29T04:31:27.281 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:26] ENGINE Bus STARTING 2024-03-29T04:31:27.281 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:27] ENGINE Serving on http://:::9283 2024-03-29T04:31:27.282 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:31:27] ENGINE Bus STARTED 2024-03-29T04:31:27.700 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":21,"active_gid":24341,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":3374290780},{"type":"v1","addr":"172.21.15.29:6801","nonce":3374290780}]},"active_addr":"172.21.15.29:6801/3374290780","active_change":"2024-03-29T04:31:09.412888+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24367,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1074832730}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":4207961916}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3638347893}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3377829364}]}]} 2024-03-29T04:31:27.703 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:31:27.704 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:31:27.704 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:31:27.973 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:31:28.096 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:31:28.096 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.096 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: Deploying daemon grafana.a on smithi099 2024-03-29T04:31:28.096 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[41899]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:28.096 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:31:28.097 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: Deploying daemon grafana.a on smithi099 2024-03-29T04:31:28.098 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:27 smithi029 ceph-mon[30044]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:28.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:31:28.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:28.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: Deploying daemon grafana.a on smithi099 2024-03-29T04:31:28.100 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:27 smithi099 ceph-mon[33865]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:28.375 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:28.152Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000408961s 2024-03-29T04:31:29.976 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:31:29.976 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":64,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:31:18.649201+0000","last_up_change":"2024-03-29T04:30:37.220270+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:31:13.255363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6803","nonce":517562633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6805","nonce":517562633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6809","nonce":517562633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6807","nonce":517562633}]},"public_addr":"172.21.15.29:6803/517562633","cluster_addr":"172.21.15.29:6805/517562633","heartbeat_back_addr":"172.21.15.29:6809/517562633","heartbeat_front_addr":"172.21.15.29:6807/517562633","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6811","nonce":1074269377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6813","nonce":1074269377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6817","nonce":1074269377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6815","nonce":1074269377}]},"public_addr":"172.21.15.29:6811/1074269377","cluster_addr":"172.21.15.29:6813/1074269377","heartbeat_back_addr":"172.21.15.29:6817/1074269377","heartbeat_front_addr":"172.21.15.29:6815/1074269377","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6819","nonce":2548163485}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6821","nonce":2548163485}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6825","nonce":2548163485}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6823","nonce":2548163485}]},"public_addr":"172.21.15.29:6819/2548163485","cluster_addr":"172.21.15.29:6821/2548163485","heartbeat_back_addr":"172.21.15.29:6825/2548163485","heartbeat_front_addr":"172.21.15.29:6823/2548163485","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6827","nonce":2031012271}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6829","nonce":2031012271}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6833","nonce":2031012271}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6831","nonce":2031012271}]},"public_addr":"172.21.15.29:6827/2031012271","cluster_addr":"172.21.15.29:6829/2031012271","heartbeat_back_addr":"172.21.15.29:6833/2031012271","heartbeat_front_addr":"172.21.15.29:6831/2031012271","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:31:30.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:30 smithi029 ceph-mon[41899]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:30.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:30 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:30.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:30 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3490333015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:30.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:30 smithi029 ceph-mon[30044]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:30.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:30 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:30.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:30 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3490333015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:30.801 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:31:30.801 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:31:30.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:30 smithi099 ceph-mon[33865]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:30.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:30 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:30.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:30 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3490333015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:31.067 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:32.602 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:32 smithi029 ceph-mon[30044]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:32.603 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:32 smithi029 ceph-mon[41899]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:32.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:32 smithi099 ceph-mon[33865]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:33.044 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:31:33.045 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":64,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:31:18.649201+0000","last_up_change":"2024-03-29T04:30:37.220270+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:31:13.255363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6803","nonce":517562633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6805","nonce":517562633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6809","nonce":517562633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6807","nonce":517562633}]},"public_addr":"172.21.15.29:6803/517562633","cluster_addr":"172.21.15.29:6805/517562633","heartbeat_back_addr":"172.21.15.29:6809/517562633","heartbeat_front_addr":"172.21.15.29:6807/517562633","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6811","nonce":1074269377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6813","nonce":1074269377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6817","nonce":1074269377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6815","nonce":1074269377}]},"public_addr":"172.21.15.29:6811/1074269377","cluster_addr":"172.21.15.29:6813/1074269377","heartbeat_back_addr":"172.21.15.29:6817/1074269377","heartbeat_front_addr":"172.21.15.29:6815/1074269377","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6819","nonce":2548163485}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6821","nonce":2548163485}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6825","nonce":2548163485}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6823","nonce":2548163485}]},"public_addr":"172.21.15.29:6819/2548163485","cluster_addr":"172.21.15.29:6821/2548163485","heartbeat_back_addr":"172.21.15.29:6825/2548163485","heartbeat_front_addr":"172.21.15.29:6823/2548163485","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6827","nonce":2031012271}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6829","nonce":2031012271}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6833","nonce":2031012271}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6831","nonce":2031012271}]},"public_addr":"172.21.15.29:6827/2031012271","cluster_addr":"172.21.15.29:6829/2031012271","heartbeat_back_addr":"172.21.15.29:6833/2031012271","heartbeat_front_addr":"172.21.15.29:6831/2031012271","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:31:33.667 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:33 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/187681342' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:33.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:33 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/187681342' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:33.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:33 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/187681342' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:31:33.873 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:31:33.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:31:33.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:31:33.874 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:31:33.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:31:33.875 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:31:33.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:31:33.876 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:31:34.453 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.453 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.453 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.454 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.454 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.458 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.458 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.459 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:34.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:34 smithi099 ceph-mon[33865]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:34.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:31:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:31:34.745 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:34 smithi029 ceph-mon[41899]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:34.745 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:34 smithi029 ceph-mon[30044]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:36.417 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:31:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:31:36.155Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003385991s 2024-03-29T04:31:36.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:36 smithi099 ceph-mon[33865]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:36.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:36 smithi029 ceph-mon[30044]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:36.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:36 smithi029 ceph-mon[41899]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:38.719 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:38 smithi029 ceph-mon[30044]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:38.719 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:38 smithi029 ceph-mon[41899]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:38.820 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:38 smithi099 ceph-mon[33865]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:39.740 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:39 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:39.741 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:39 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:39.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:39 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:40.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:40 smithi099 ceph-mon[33865]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:40.628 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:40 smithi029 ceph-mon[41899]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:40.629 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:40 smithi029 ceph-mon[30044]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:40.654 INFO:teuthology.orchestra.run.smithi029.stdout:167503724568 2024-03-29T04:31:40.654 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:31:41.077 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:40 smithi099 podman[65519]: 2024-03-29 04:31:40.979577864 +0000 UTC m=+0.138299728 container create 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, name=ubi8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.buildah.version=1.32.0, architecture=x86_64, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.openshift.expose-services=, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 podman[65519]: 2024-03-29 04:31:41.223464678 +0000 UTC m=+0.382186512 container init 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, version=8.5) 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 podman[65519]: 2024-03-29 04:31:41.229908858 +0000 UTC m=+0.388630680 container start 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, version=8.5, vendor=Red Hat, Inc., description=Ceph Grafana Container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, architecture=x86_64, name=ubi8, maintainer=Paul Cuzner , io.buildah.version=1.32.0) 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 bash[65519]: 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381606023Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381780907Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-03-29T04:31:41.449 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381799818Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-03-29T04:31:41.450 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381807057Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-03-29T04:31:41.450 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.3818132Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-03-29T04:31:41.450 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381820616Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-03-29T04:31:41.450 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.38182644Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-03-29T04:31:41.450 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.38183258Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-03-29T04:31:41.450 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381838761Z level=info msg="Path Home" path=/usr/share/grafana 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381844828Z level=info msg="Path Data" path=/var/lib/grafana 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381850459Z level=info msg="Path Logs" path=/var/log/grafana 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381856224Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.381862086Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=settings t=2024-03-29T04:31:41.38186801Z level=info msg="App mode production" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore t=2024-03-29T04:31:41.381912246Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore t=2024-03-29T04:31:41.381928523Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.382748856Z level=info msg="Starting DB migrations" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.382821815Z level=info msg="Executing migration" id="create migration_log table" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.383338871Z level=info msg="Executing migration" id="create user table" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.383765815Z level=info msg="Executing migration" id="add unique index user.login" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.384227163Z level=info msg="Executing migration" id="add unique index user.email" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.384804604Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-03-29T04:31:41.451 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.385236192Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.385619705Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.386432493Z level=info msg="Executing migration" id="create user table v2" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.38684625Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.38769003Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.388269833Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.38864162Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.388985851Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.389412927Z level=info msg="Executing migration" id="Update user table charset" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.389702082Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.390150625Z level=info msg="Executing migration" id="Add missing user data" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.390517657Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.390913827Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.391334188Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-03-29T04:31:41.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.391740557Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-03-29T04:31:41.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.39286095Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-03-29T04:31:41.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.39336135Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-03-29T04:31:41.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.393816649Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-03-29T04:31:41.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.394265394Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-03-29T04:31:41.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.394668925Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-03-29T04:31:41.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.395061632Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.395387113Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.395746775Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.396141107Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.39652893Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.396889604Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.397556381Z level=info msg="Executing migration" id="create temp_user v2" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.397923978Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.398388467Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.398754812Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.399160911Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.399597254Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-03-29T04:31:41.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.399913813Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.400271505Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.400595613Z level=info msg="Executing migration" id="create star table" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.400938216Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.4013996Z level=info msg="Executing migration" id="create org table v1" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.401781121Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.402195095Z level=info msg="Executing migration" id="create org_user table v1" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.402590167Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.402998369Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.403439796Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.403823204Z level=info msg="Executing migration" id="Update org table charset" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.404122003Z level=info msg="Executing migration" id="Update org_user table charset" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.404436266Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.404735408Z level=info msg="Executing migration" id="create dashboard table" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.405111403Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-03-29T04:31:41.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.405540108Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.405938492Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.406385732Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.407245788Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.408185201Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.4090552Z level=info msg="Executing migration" id="create dashboard v2" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.409920285Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.410943161Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.411819517Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.412336766Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.412727596Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.4130143Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.413556983Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.41401562Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-03-29T04:31:41.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.414546417Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.416890701Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.417435588Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.419754132Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.420243109Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.420564635Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.420834948Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-03-29T04:31:41.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.421391064Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.424339918Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.42688449Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.427491839Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.430472387Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.431104046Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.433134726Z level=info msg="Executing migration" id="Update dashboard title length" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.436041141Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.436537459Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.439286527Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.442168283Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.444715197Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.447454882Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.448004469Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-03-29T04:31:41.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.450690696Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.453300778Z level=info msg="Executing migration" id="Add check_sum column" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.456156381Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.456704941Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.45949374Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.45989384Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.462291731Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.464831395Z level=info msg="Executing migration" id="create data_source table" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.465297129Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.467985196Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.468419186Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.47115943Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.471684565Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.4741514Z level=info msg="Executing migration" id="create data_source table v2" 2024-03-29T04:31:41.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.476550177Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-03-29T04:31:41.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.479199876Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-03-29T04:31:41.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.480043689Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-03-29T04:31:41.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.480428403Z level=info msg="Executing migration" id="Add column with_credentials" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.481240421Z level=info msg="Executing migration" id="Add secure json data column" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.481795093Z level=info msg="Executing migration" id="Update data_source table charset" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.48210765Z level=info msg="Executing migration" id="Update initial version to 1" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.482973286Z level=info msg="Executing migration" id="Add read_only data column" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.48351542Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.484307887Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.484631548Z level=info msg="Executing migration" id="Add uid column" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.485183501Z level=info msg="Executing migration" id="Update uid value" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.486031018Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.486465375Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.487451004Z level=info msg="Executing migration" id="create api_key table" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.488140171Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-03-29T04:31:41.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.489121423Z level=info msg="Executing migration" id="add index api_key.key" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.489711384Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.490587651Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.491548279Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.492359861Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.493133469Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.494284571Z level=info msg="Executing migration" id="create api_key table v2" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.495163678Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.496179176Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.497140498Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.497840691Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.49879983Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.499538078Z level=info msg="Executing migration" id="Update api_key table charset" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.500442547Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.501147328Z level=info msg="Executing migration" id="Add service account foreign key" 2024-03-29T04:31:41.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.501739286Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.502538994Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.503110892Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.503733452Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.504527383Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.505450482Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.506280467Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-03-29T04:31:41.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.509234931Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-03-29T04:31:41.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.511976564Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-03-29T04:31:41.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.514884405Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.517690896Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.520434355Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.523332866Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.526131721Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.529038449Z level=info msg="Executing migration" id="create quota table v1" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.5320351Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.534772512Z level=info msg="Executing migration" id="Update quota table charset" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.537633334Z level=info msg="Executing migration" id="create plugin_setting table" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.540607842Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.543467606Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.546274085Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.549179937Z level=info msg="Executing migration" id="create session table" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.552103644Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-03-29T04:31:41.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.554981445Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.557483119Z level=info msg="Executing migration" id="create playlist table v2" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.560084903Z level=info msg="Executing migration" id="create playlist item table v2" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.562869832Z level=info msg="Executing migration" id="Update playlist table charset" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.565342619Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.568045707Z level=info msg="Executing migration" id="drop preferences table v2" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.570740149Z level=info msg="Executing migration" id="drop preferences table v3" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.573384774Z level=info msg="Executing migration" id="create preferences table v3" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.576164905Z level=info msg="Executing migration" id="Update preferences table charset" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.578847639Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.581418053Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.584185404Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.586981686Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.589703815Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.592479291Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.595375795Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-03-29T04:31:41.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.598149209Z level=info msg="Executing migration" id="create alert table v1" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.600597652Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.603254924Z level=info msg="Executing migration" id="add index alert state" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.604083026Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.605055135Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.605915139Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.606648792Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.607490127Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.609082603Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.609879764Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.610895128Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.611642274Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.612436867Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.612881584Z level=info msg="Executing migration" id="Add column is_default" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.613548651Z level=info msg="Executing migration" id="Add column frequency" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.614401638Z level=info msg="Executing migration" id="Add column send_reminder" 2024-03-29T04:31:41.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.615350329Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.616159344Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.617221153Z level=info msg="Executing migration" id="Update alert table charset" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.617921693Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.618808747Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.619698561Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.620544461Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.621437654Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.622299975Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.623322787Z level=info msg="Executing migration" id="Add for to alert table" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.624107683Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.625164886Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.625904503Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-03-29T04:31:41.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.626911524Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.627641239Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.628520325Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.629540066Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.630314303Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.631250244Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.631974814Z level=info msg="Executing migration" id="create annotation table v5" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.632829308Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.633673309Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.636545069Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.639284317Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.642303723Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.645176094Z level=info msg="Executing migration" id="Update annotation table charset" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.647812215Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.650685213Z level=info msg="Executing migration" id="Drop category_id index" 2024-03-29T04:31:41.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.653505674Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.656295848Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.659058798Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.662080187Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.664782833Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.667713942Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.670645467Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.673555984Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.676341911Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.679070293Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.681957197Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.684440845Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.687065085Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.689662754Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-03-29T04:31:41.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.69217653Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.694923028Z level=info msg="Executing migration" id="Add epoch_end column" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.697617878Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.700352314Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.703067136Z level=info msg="Executing migration" id="Move region to single row" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.70550976Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.708324096Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.710996493Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.713756675Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.716445708Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.719228229Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.721913619Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.724475172Z level=info msg="Executing migration" id="create test_data table" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.727236897Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-03-29T04:31:41.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.729787602Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.730664501Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.73151516Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.732356526Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.733214913Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.734017474Z level=info msg="Executing migration" id="create team table" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.734850851Z level=info msg="Executing migration" id="add index team.org_id" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.735335264Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.736193085Z level=info msg="Executing migration" id="create team member table" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.73704548Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-03-29T04:31:41.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.738009584Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.738745707Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.739773081Z level=info msg="Executing migration" id="Add column email to team table" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.740576191Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.741432969Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.744530792Z level=info msg="Executing migration" id="create dashboard acl table" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.747287007Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.750062339Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.750503102Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.752908928Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.755803427Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.758595731Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.759056357Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.761936987Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-03-29T04:31:41.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.764547888Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.767348085Z level=info msg="Executing migration" id="create tag table" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.767750051Z level=info msg="Executing migration" id="add index tag.key_value" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.770691896Z level=info msg="Executing migration" id="create login attempt table" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.773417876Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.776328653Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.779310913Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.782069552Z level=info msg="Executing migration" id="create login_attempt v2" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.784735185Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.787496537Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.787860718Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.790625984Z level=info msg="Executing migration" id="create user auth table" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.793385562Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.795922476Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-03-29T04:31:41.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.796394594Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.799104728Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.801906976Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.804516126Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.805364268Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.808040396Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.810614865Z level=info msg="Executing migration" id="create server_lock table" 2024-03-29T04:31:41.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.813417889Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.813980425Z level=info msg="Executing migration" id="create user auth token table" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.817003579Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.819884819Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.822371676Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.822835299Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.823691886Z level=info msg="Executing migration" id="create cache_data table" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.824517154Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.82548919Z level=info msg="Executing migration" id="create short_url table v1" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.826179972Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.826940918Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.827754582Z level=info msg="Executing migration" id="delete alert_definition table" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.828049429Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.828488099Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.829154038Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-03-29T04:31:41.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.829629572Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.829921208Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.830355886Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.830762619Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.831223261Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.831661831Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.832494898Z level=info msg="Executing migration" id="drop alert_definition table" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.832911491Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.833239591Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.833643938Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.834112481Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.834556674Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.834850589Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.835283241Z level=info msg="Executing migration" id="create alert_instance table" 2024-03-29T04:31:41.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.835719218Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.83619977Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.836647504Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.83747144Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.837904458Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.838357096Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.84179965Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.84519175Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.845660483Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.846110857Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.846943196Z level=info msg="Executing migration" id="create alert_rule table" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.847402321Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.84784675Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-03-29T04:31:41.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.848317425Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.848763421Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.849091841Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.849907111Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.850743617Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.851598308Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.852036628Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.852479825Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.853304893Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.854151839Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.854602501Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.855447237Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.856308761Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.856636555Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.85713933Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-03-29T04:31:41.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.857588832Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.858036852Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.85835724Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.859248532Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.860109624Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.860983396Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.861840778Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.862740357Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.863697728Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.864237884Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.865103196Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.866003023Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.866980153Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.867438943Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-03-29T04:31:41.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.86830925Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.869279418Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.870091512Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.871117698Z level=info msg="Executing migration" id="create provenance_type table" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.872028407Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.873061143Z level=info msg="Executing migration" id="create alert_image table" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.873686025Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.874151119Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.875040263Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.875813623Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.876249335Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.876351063Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.876646075Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.877575655Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.878125876Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-03-29T04:31:41.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.87826484Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.878393843Z level=info msg="alerts found to migrate" alerts=0 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.878807888Z level=info msg="Executing migration" id="create library_element table v1" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.879680716Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.880169797Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.883237368Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.886278864Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.888986034Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.889412612Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.892104843Z level=info msg="Executing migration" id="create data_keys table" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.892529179Z level=info msg="Executing migration" id="create secrets table" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.895242181Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.899622504Z level=info msg="Executing migration" id="add name column into data_keys" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.900544535Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-03-29T04:31:41.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.901414024Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.905678409Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.909899029Z level=info msg="Executing migration" id="create kv_store table v1" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.910345216Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.911265797Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.912038049Z level=info msg="Executing migration" id="create permission table" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.912857389Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.91332642Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.914178051Z level=info msg="Executing migration" id="create role table" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.915125808Z level=info msg="Executing migration" id="add column display_name" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.916220456Z level=info msg="Executing migration" id="add column group_name" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.91717486Z level=info msg="Executing migration" id="add index role.org_id" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.918075959Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.918797953Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-03-29T04:31:41.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.919782384Z level=info msg="Executing migration" id="create team role table" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.920348022Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.921292667Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.922055927Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.923062534Z level=info msg="Executing migration" id="create user role table" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.92353845Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.924446362Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.925287485Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.926146919Z level=info msg="Executing migration" id="create builtin role table" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.92657404Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.927495799Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.928328261Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.929318071Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.929792177Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-03-29T04:31:41.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.930826521Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.93153515Z level=info msg="Executing migration" id="add unique index role.uid" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.932454739Z level=info msg="Executing migration" id="create seed assignment table" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.932993501Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.933459603Z level=info msg="Executing migration" id="add column hidden to role table" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.934447115Z level=info msg="Executing migration" id="create query_history table v1" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.93746759Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.940270028Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.940590072Z level=info msg="Executing migration" id="teams permissions migration" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.943575935Z level=info msg="Executing migration" id="dashboard permissions" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.944062532Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.946349624Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.949264495Z level=info msg="Executing migration" id="alerting notification permissions" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.949631899Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-03-29T04:31:41.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.952467203Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.955334243Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.958183934Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.958523271Z level=info msg="Executing migration" id="create correlation table v1" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.961564772Z level=info msg="Executing migration" id="add index correlations.uid" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.964592222Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.96739168Z level=info msg="Executing migration" id="add correlation config column" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.9684621Z level=info msg="Executing migration" id="create entity_events table" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.971380945Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.97409452Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.974200838Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.976640214Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.976734318Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.977024998Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-03-29T04:31:41.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.979735611Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-03-29T04:31:41.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.98228361Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:31:42.348 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.984974237Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:31:42.348 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.985427377Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-03-29T04:31:42.348 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.988044254Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-29T04:31:42.348 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.98848534Z level=info msg="Executing migration" id="Drop public config table" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.990642707Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.991070562Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.99353664Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.994049675Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.996825316Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:41 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:41.999834517Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.0024203Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.003405476Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.006102599Z level=info msg="Executing migration" id="add share column" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.007095906Z level=info msg="Executing migration" id="create default alerting folders" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.00794671Z level=info msg="Executing migration" id="create file table" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.008404256Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-03-29T04:31:42.349 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.009212904Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.010054323Z level=info msg="Executing migration" id="create file_meta table" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.010893502Z level=info msg="Executing migration" id="file table idx: path key" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.011367055Z level=info msg="Executing migration" id="set path collation in file table" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.012258039Z level=info msg="Executing migration" id="managed permissions migration" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.012871146Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.013203444Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.014170787Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.015358854Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.016307442Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.01697401Z level=info msg="Executing migration" id="update group index for alert rules" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.017828258Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.018196263Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.019147969Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-03-29T04:31:42.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.020296783Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.021325646Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.021790338Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.031552065Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.032410206Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.033223462Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.036615257Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.037444413Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.038349707Z level=info msg="Executing migration" id="create folder table" 2024-03-29T04:31:42.351 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.039155952Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.039998965Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.040901057Z level=info msg="Executing migration" id="Update folder title length" 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.041771411Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=migrator t=2024-03-29T04:31:42.042661141Z level=info msg="migrations completed" performed=485 skipped=0 duration=659.855198ms 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore t=2024-03-29T04:31:42.043278593Z level=info msg="Created default organization" 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=plugin.loader t=2024-03-29T04:31:42.097900486Z level=info msg="Plugin registered" pluginID=input 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=plugin.loader t=2024-03-29T04:31:42.126360996Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=plugin.loader t=2024-03-29T04:31:42.126392588Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=secrets t=2024-03-29T04:31:42.12642622Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=query_data t=2024-03-29T04:31:42.128123331Z level=info msg="Query Service initialization" 2024-03-29T04:31:42.352 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=live.push_http t=2024-03-29T04:31:42.130767481Z level=info msg="Live Push Gateway initialization" 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[41899]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[41899]: Deploying daemon node-exporter.a on smithi029 2024-03-29T04:31:42.625 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[30044]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:42.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.626 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:42 smithi029 ceph-mon[30044]: Deploying daemon node-exporter.a on smithi029 2024-03-29T04:31:42.817 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.818 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.818 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-mon[33865]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:42.818 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.818 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:42.818 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-mon[33865]: Deploying daemon node-exporter.a on smithi029 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=infra.usagestats.collector t=2024-03-29T04:31:42.843247933Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=provisioning.datasources t=2024-03-29T04:31:42.843572806Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=provisioning.datasources t=2024-03-29T04:31:42.876364705Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=provisioning.datasources t=2024-03-29T04:31:42.907708025Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=provisioning.alerting t=2024-03-29T04:31:42.929357435Z level=info msg="starting to provision alerting" 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=provisioning.alerting t=2024-03-29T04:31:42.929376507Z level=info msg="finished to provision alerting" 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=ngalert.state.manager t=2024-03-29T04:31:42.929453093Z level=info msg="Warming state cache for startup" 2024-03-29T04:31:43.098 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=grafanaStorageLogger t=2024-03-29T04:31:42.92958133Z level=info msg="storage starting" 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=http.server t=2024-03-29T04:31:42.931439367Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore.transactions t=2024-03-29T04:31:42.940410709Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=ngalert.state.manager t=2024-03-29T04:31:42.940431921Z level=info msg="State cache has been initialized" states=0 duration=10.9778ms 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=ngalert.multiorg.alertmanager t=2024-03-29T04:31:42.940490027Z level=info msg="starting MultiOrg Alertmanager" 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=ticker t=2024-03-29T04:31:42.940525586Z level=info msg=starting first_tick=2024-03-29T04:31:50Z 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore.transactions t=2024-03-29T04:31:42.978976805Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore.transactions t=2024-03-29T04:31:43.01984847Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:31:43.099 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:31:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[65533]: logger=sqlstore.transactions t=2024-03-29T04:31:43.030322763Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-03-29T04:31:43.168 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:43 smithi029 systemd[1]: Starting Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:31:44.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:44 smithi099 ceph-mon[33865]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:44.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:31:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:31:44.732 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:44 smithi029 ceph-mon[30044]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:44.732 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:44 smithi029 ceph-mon[41899]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:45.084 INFO:teuthology.orchestra.run.smithi029.stdout:34359738419 2024-03-29T04:31:45.084 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:31:45.667 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:45 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:45.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:45 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:45.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:45 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:46.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:46 smithi099 ceph-mon[33865]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:46.917 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:46 smithi029 ceph-mon[30044]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:46.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:46 smithi029 ceph-mon[41899]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:48.667 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:48 smithi029 ceph-mon[41899]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:48.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:48 smithi029 ceph-mon[30044]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:48.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:48 smithi099 ceph-mon[33865]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:49.952 INFO:teuthology.orchestra.run.smithi029.stdout:236223201295 2024-03-29T04:31:49.952 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:31:50.410 INFO:teuthology.orchestra.run.smithi029.stdout:201863462934 2024-03-29T04:31:50.410 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:31:50.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:50 smithi029 ceph-mon[30044]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:50.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:50 smithi029 ceph-mon[41899]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:50.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:50 smithi099 ceph-mon[33865]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:50.887 INFO:teuthology.orchestra.run.smithi029.stdout:133143986207 2024-03-29T04:31:50.887 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:31:51.281 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:50 smithi029 bash[75409]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-03-29T04:31:51.295 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:51.307 INFO:teuthology.orchestra.run.smithi029.stdout:111669149732 2024-03-29T04:31:51.308 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:31:51.595 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:51 smithi029 bash[75409]: Getting image source signatures 2024-03-29T04:31:51.596 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:51 smithi029 bash[75409]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-03-29T04:31:51.596 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:51 smithi029 bash[75409]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-03-29T04:31:51.596 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:51 smithi029 bash[75409]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-03-29T04:31:52.343 INFO:teuthology.orchestra.run.smithi029.stdout:77309411369 2024-03-29T04:31:52.343 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:31:52.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:52 smithi099 ceph-mon[33865]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:52.853 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:52 smithi029 ceph-mon[30044]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:52.854 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:52 smithi029 ceph-mon[41899]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:53.668 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:53 smithi029 bash[75409]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-03-29T04:31:54.006 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:53 smithi029 bash[75409]: Writing manifest to image destination 2024-03-29T04:31:54.411 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:54.418 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:54 smithi029 podman[75409]: 2024-03-29 04:31:54.177861763 +0000 UTC m=+3.741319920 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:31:54.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:54 smithi099 ceph-mon[33865]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:54.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:54 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:54.599 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:31:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:31:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:31:54.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:54 smithi029 ceph-mon[30044]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:54.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:54 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:54.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:54 smithi029 ceph-mon[41899]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:54.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:54 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:31:54.738 INFO:teuthology.orchestra.run.smithi029.stdout:55834574895 2024-03-29T04:31:54.739 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:31:54.861 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:55.288 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:55.540 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:55 smithi029 podman[75409]: 2024-03-29 04:31:55.408632073 +0000 UTC m=+4.972090260 container create 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:31:56.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:56 smithi029 ceph-mon[30044]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:56.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:56 smithi029 ceph-mon[41899]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:56.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:56 smithi099 ceph-mon[33865]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:57.668 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 podman[75409]: 2024-03-29 04:31:57.515669793 +0000 UTC m=+7.079127950 container init 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 podman[75409]: 2024-03-29 04:31:57.522179721 +0000 UTC m=+7.085637866 container start 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.526Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.526Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:31:57.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:31:57.670 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:31:57.671 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:31:57.672 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:31:57.673 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.527Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:31:57.673 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.528Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:31:57.673 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[75968]: ts=2024-03-29T04:31:57.528Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:31:57.673 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 bash[75409]: 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 2024-03-29T04:31:57.673 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:31:57 smithi029 systemd[1]: Started Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:31:58.302 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:58.302 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:31:58.435 INFO:teuthology.orchestra.run.smithi029.stdout:167503724572 2024-03-29T04:31:58.482 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:31:58 smithi099 systemd[1]: Starting Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:31:58.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:58 smithi099 ceph-mon[33865]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:58.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:58 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:58 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:58 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:58 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2774238583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:31:58.849 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:31:58 smithi099 bash[65818]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-03-29T04:31:58.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[30044]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:58.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2774238583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:31:58.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[41899]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:31:58.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:31:58.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:58 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2774238583' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:31:59.099 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724568 got 167503724572 for osd.5 2024-03-29T04:31:59.099 DEBUG:teuthology.parallel:result is None 2024-03-29T04:31:59.348 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:31:58 smithi099 bash[65818]: Getting image source signatures 2024-03-29T04:31:59.348 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:31:58 smithi099 bash[65818]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-03-29T04:31:59.348 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:31:58 smithi099 bash[65818]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-03-29T04:31:59.348 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:31:58 smithi099 bash[65818]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-03-29T04:31:59.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:31:59 smithi099 ceph-mon[33865]: Deploying daemon node-exporter.b on smithi099 2024-03-29T04:31:59.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:31:59 smithi029 ceph-mon[30044]: Deploying daemon node-exporter.b on smithi099 2024-03-29T04:31:59.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:31:59 smithi029 ceph-mon[41899]: Deploying daemon node-exporter.b on smithi099 2024-03-29T04:32:00.750 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:00 smithi099 ceph-mon[33865]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:00.750 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:00 smithi099 bash[65818]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-03-29T04:32:00.750 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:00 smithi099 bash[65818]: Writing manifest to image destination 2024-03-29T04:32:00.806 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:00 smithi029 ceph-mon[30044]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:00.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:00 smithi029 ceph-mon[41899]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:01.032 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 podman[65818]: 2024-03-29 04:32:00.959226989 +0000 UTC m=+2.467398142 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:32:01.032 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 podman[65818]: 2024-03-29 04:32:01.056860929 +0000 UTC m=+2.565032071 container create 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:32:01.599 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 podman[65818]: 2024-03-29 04:32:01.309455041 +0000 UTC m=+2.817626196 container init 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:32:01.599 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 podman[65818]: 2024-03-29 04:32:01.315022783 +0000 UTC m=+2.823193928 container start 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:32:01.599 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.318Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:32:01.600 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.318Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:32:01.600 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.319Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:32:01.600 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.319Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:32:01.600 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.319Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:32:01.600 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.319Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:32:01.600 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.319Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:32:01.601 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:32:01.601 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:32:01.601 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:32:01.601 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:32:01.601 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:32:01.601 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:32:01.602 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:32:01.603 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:32:01.603 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:32:01.603 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:32:01.603 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:32:01.603 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:32:01.603 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:32:01.604 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:32:01.604 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:32:01.604 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:32:01.604 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:32:01.604 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:32:01.604 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:32:01.605 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:32:01.605 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:32:01.605 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:32:01.605 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:32:01.605 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:32:01.605 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:32:01.606 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:32:01.606 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:32:01.606 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:32:01.606 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:32:01.606 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:32:01.606 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:32:01.607 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:32:01.607 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:32:01.607 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:32:01.607 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:32:01.607 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:32:01.608 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:32:01.608 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:32:01.608 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:32:01.608 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:32:01.608 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:32:01.608 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:32:01.609 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:32:01.609 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:32:01.609 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:32:01.609 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[65868]: ts=2024-03-29T04:32:01.320Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:32:01.609 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 bash[65818]: 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 2024-03-29T04:32:01.609 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:32:01 smithi099 systemd[1]: Started Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:32:02.000 INFO:teuthology.orchestra.run.smithi029.stdout:34359738423 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:02.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:02 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3154360392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3154360392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:32:02.910 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:02.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:02 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3154360392' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:32:03.950 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:04.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:04 smithi099 ceph-mon[33865]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:04.599 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:32:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:32:04.722 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:04 smithi029 ceph-mon[30044]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:04.723 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:04 smithi029 ceph-mon[41899]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:04.798 INFO:teuthology.orchestra.run.smithi029.stdout:133143986210 2024-03-29T04:32:05.491 INFO:teuthology.orchestra.run.smithi029.stdout:236223201299 2024-03-29T04:32:05.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2857124690' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:32:05.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2379295243' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:32:05.840 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.840 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.840 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2857124690' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:32:05.840 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.840 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2379295243' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:32:05.840 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:05 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:05 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2857124690' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:32:05.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:05 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:05.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2379295243' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:32:05.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:05 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:06.157 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201299 for osd.7 2024-03-29T04:32:06.157 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:06.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:06 smithi099 ceph-mon[33865]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:06.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:06 smithi029 ceph-mon[30044]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:06.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:06 smithi029 ceph-mon[41899]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:06.968 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986207 got 133143986210 for osd.4 2024-03-29T04:32:06.969 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:07.389 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:07.419 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738419 got 34359738423 for osd.0 2024-03-29T04:32:07.420 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:08.304 INFO:teuthology.orchestra.run.smithi029.stdout:77309411373 2024-03-29T04:32:08.383 INFO:teuthology.orchestra.run.smithi029.stdout:201863462938 2024-03-29T04:32:08.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:08 smithi099 ceph-mon[33865]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:08.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:08 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/115378870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:32:08.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:08 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/869953368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:32:08.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:08 smithi029 ceph-mon[30044]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:08.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:08 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/115378870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:32:08.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:08 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/869953368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:32:08.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:08 smithi029 ceph-mon[41899]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:08.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:08 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/115378870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:32:08.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:08 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/869953368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:32:09.663 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:09.663 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:09.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:10.010 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411369 got 77309411373 for osd.2 2024-03-29T04:32:10.010 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:10.587 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:10 smithi029 ceph-mon[30044]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:10.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:10 smithi029 ceph-mon[41899]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:10.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:10 smithi099 ceph-mon[33865]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:11.431 INFO:teuthology.orchestra.run.smithi029.stdout:55834574898 2024-03-29T04:32:11.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:11 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2164501591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:32:11.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:11 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2164501591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:32:11.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:11 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2164501591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:32:12.085 INFO:teuthology.orchestra.run.smithi029.stdout:111669149737 2024-03-29T04:32:12.349 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462934 got 201863462938 for osd.6 2024-03-29T04:32:12.349 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:12.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:12 smithi029 ceph-mon[41899]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:12.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:12 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3361323707' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:32:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:12 smithi029 ceph-mon[30044]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:12.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:12 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3361323707' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:32:12.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:12 smithi099 ceph-mon[33865]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:12.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:12 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3361323707' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:32:12.962 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149732 got 111669149737 for osd.3 2024-03-29T04:32:12.962 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:13.447 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574895 got 55834574898 for osd.1 2024-03-29T04:32:13.447 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:13.448 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:32:13.448 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:32:14.183 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:14.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:14 smithi099 ceph-mon[33865]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:14.600 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:32:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:32:14.615 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:14 smithi029 ceph-mon[30044]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:14.615 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:14 smithi029 ceph-mon[41899]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:16.080 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:32:16.082 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-03-29T04:32:16.625 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:16 smithi029 ceph-mon[30044]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:16.625 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:16 smithi029 ceph-mon[41899]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:16.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:16 smithi099 ceph-mon[33865]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:17.012 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-03-29T04:32:15.434369+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385548,"kb_used_data":6696,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575668,"statfs":{"total":767960285184,"available":765517484032,"internally_reserved":0,"allocated":6856704,"data_stored":3899686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002013"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548406+0000","last_change":"2024-03-29T04:31:14.925966+0000","last_active":"2024-03-29T04:31:16.548406+0000","last_peered":"2024-03-29T04:31:16.548406+0000","last_clean":"2024-03-29T04:31:16.548406+0000","last_became_active":"2024-03-29T04:31:14.925528+0000","last_became_peered":"2024-03-29T04:31:14.925528+0000","last_unstale":"2024-03-29T04:31:16.548406+0000","last_undegraded":"2024-03-29T04:31:16.548406+0000","last_fullsized":"2024-03-29T04:31:16.548406+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:30:41.616775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548134+0000","last_change":"2024-03-29T04:31:15.188305+0000","last_active":"2024-03-29T04:31:16.548134+0000","last_peered":"2024-03-29T04:31:16.548134+0000","last_clean":"2024-03-29T04:31:16.548134+0000","last_became_active":"2024-03-29T04:31:15.187906+0000","last_became_peered":"2024-03-29T04:31:15.187906+0000","last_unstale":"2024-03-29T04:31:16.548134+0000","last_undegraded":"2024-03-29T04:31:16.548134+0000","last_fullsized":"2024-03-29T04:31:16.548134+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:03:13.618806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924358+0000","last_change":"2024-03-29T04:31:15.188174+0000","last_active":"2024-03-29T04:31:15.924358+0000","last_peered":"2024-03-29T04:31:15.924358+0000","last_clean":"2024-03-29T04:31:15.924358+0000","last_became_active":"2024-03-29T04:31:15.187629+0000","last_became_peered":"2024-03-29T04:31:15.187629+0000","last_unstale":"2024-03-29T04:31:15.924358+0000","last_undegraded":"2024-03-29T04:31:15.924358+0000","last_fullsized":"2024-03-29T04:31:15.924358+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:47:42.088021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924354+0000","last_change":"2024-03-29T04:31:14.926407+0000","last_active":"2024-03-29T04:31:15.924354+0000","last_peered":"2024-03-29T04:31:15.924354+0000","last_clean":"2024-03-29T04:31:15.924354+0000","last_became_active":"2024-03-29T04:31:14.926060+0000","last_became_peered":"2024-03-29T04:31:14.926060+0000","last_unstale":"2024-03-29T04:31:15.924354+0000","last_undegraded":"2024-03-29T04:31:15.924354+0000","last_fullsized":"2024-03-29T04:31:15.924354+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:30.113952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924418+0000","last_change":"2024-03-29T04:30:40.532165+0000","last_active":"2024-03-29T04:31:15.924418+0000","last_peered":"2024-03-29T04:31:15.924418+0000","last_clean":"2024-03-29T04:31:15.924418+0000","last_became_active":"2024-03-29T04:30:40.531726+0000","last_became_peered":"2024-03-29T04:30:40.531726+0000","last_unstale":"2024-03-29T04:31:15.924418+0000","last_undegraded":"2024-03-29T04:31:15.924418+0000","last_fullsized":"2024-03-29T04:31:15.924418+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_clean_scrub_stamp":"2024-03-29T04:28:33.017662+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:56.360284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548399+0000","last_change":"2024-03-29T04:31:14.925369+0000","last_active":"2024-03-29T04:31:16.548399+0000","last_peered":"2024-03-29T04:31:16.548399+0000","last_clean":"2024-03-29T04:31:16.548399+0000","last_became_active":"2024-03-29T04:31:14.924765+0000","last_became_peered":"2024-03-29T04:31:14.924765+0000","last_unstale":"2024-03-29T04:31:16.548399+0000","last_undegraded":"2024-03-29T04:31:16.548399+0000","last_fullsized":"2024-03-29T04:31:16.548399+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:59:12.817095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924014+0000","last_change":"2024-03-29T04:31:14.925939+0000","last_active":"2024-03-29T04:31:15.924014+0000","last_peered":"2024-03-29T04:31:15.924014+0000","last_clean":"2024-03-29T04:31:15.924014+0000","last_became_active":"2024-03-29T04:31:14.925609+0000","last_became_peered":"2024-03-29T04:31:14.925609+0000","last_unstale":"2024-03-29T04:31:15.924014+0000","last_undegraded":"2024-03-29T04:31:15.924014+0000","last_fullsized":"2024-03-29T04:31:15.924014+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:51.860391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548386+0000","last_change":"2024-03-29T04:31:14.924983+0000","last_active":"2024-03-29T04:31:16.548386+0000","last_peered":"2024-03-29T04:31:16.548386+0000","last_clean":"2024-03-29T04:31:16.548386+0000","last_became_active":"2024-03-29T04:31:14.924603+0000","last_became_peered":"2024-03-29T04:31:14.924603+0000","last_unstale":"2024-03-29T04:31:16.548386+0000","last_undegraded":"2024-03-29T04:31:16.548386+0000","last_fullsized":"2024-03-29T04:31:16.548386+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:39:19.333479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548153+0000","last_change":"2024-03-29T04:31:14.925317+0000","last_active":"2024-03-29T04:31:16.548153+0000","last_peered":"2024-03-29T04:31:16.548153+0000","last_clean":"2024-03-29T04:31:16.548153+0000","last_became_active":"2024-03-29T04:31:14.925065+0000","last_became_peered":"2024-03-29T04:31:14.925065+0000","last_unstale":"2024-03-29T04:31:16.548153+0000","last_undegraded":"2024-03-29T04:31:16.548153+0000","last_fullsized":"2024-03-29T04:31:16.548153+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:11.187076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39700000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.52800000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.41199999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.505},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51200000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.375}]},{"osd":5,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.441}]},{"osd":6,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462940,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70299999999999996,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55200000000000005,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.76100000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.67700000000000005}]},{"osd":1,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77600000000000002,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.57499999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.752,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.46800000000000003,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78300000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.46800000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53100000000000003,"15min":0.53000000000000003},"min":{"1min":0.42899999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79300000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.71299999999999997,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.46800000000000003,"15min":0.46600000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61799999999999999,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.59399999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.49099999999999999,"15min":0.48099999999999998},"min":{"1min":0.40699999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.63},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.48999999999999999,"15min":0.48399999999999999},"min":{"1min":0.376,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.64500000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.71599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574900,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.35499999999999998,"15min":0.33200000000000002},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.36799999999999999,"15min":0.34000000000000002},"min":{"1min":0.252,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.38}]},{"osd":2,"last update":"Fri Mar 29 04:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.28199999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.52000000000000002,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.45000000000000001,"15min":0.441},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56399999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42399999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.42599999999999999,"15min":0.439},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.51000000000000001,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.46000000000000002,"15min":0.46899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.432}]},{"osd":4,"last update":"Fri Mar 29 04:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70299999999999996,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.52900000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:31:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.499},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77400000000000002,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.63300000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:32:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.55300000000000005,"15min":0.54700000000000004},"min":{"1min":0.41199999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.75700000000000001,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.59799999999999998,"15min":0.59199999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88800000000000001,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.46800000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:31:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738426,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:32:14 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.35799999999999998,"15min":0.33600000000000002},"min":{"1min":0.26200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.39100000000000001,"15min":0.35699999999999998},"min":{"1min":0.28299999999999997,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.61099999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.45500000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:31:39 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.65200000000000002,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40600000000000003,"15min":0.40899999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53500000000000003,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.61399999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:32:03 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.48199999999999998,"15min":0.48999999999999999},"min":{"1min":0.316,"5min":0.315,"15min":0.315},"max":{"1min":0.53000000000000003,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.46899999999999997,"15min":0.47699999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.54500000000000004,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48899999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.90600000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.68600000000000005,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.64600000000000002,"15min":0.66000000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.90600000000000003,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.94199999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:32:14 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.626,"15min":0.629},"min":{"1min":0.48899999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80700000000000005,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59799999999999998,"15min":0.59499999999999997},"min":{"1min":0.42299999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.89500000000000002,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.89500000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.85199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.38100000000000001,"15min":0.36899999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53600000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40400000000000003,"15min":0.38800000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.56699999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.371}]},{"osd":1,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42999999999999999,"15min":0.41999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41899999999999998,"15min":0.41199999999999998},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.58199999999999996,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.253}]},{"osd":3,"last update":"Fri Mar 29 04:32:04 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.45200000000000001,"15min":0.47699999999999998},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.48299999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.51100000000000001,"15min":0.53000000000000003},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.65200000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38500000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.45900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.69399999999999995,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.66200000000000003,"15min":0.66500000000000004},"min":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.58799999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:31:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.623,"15min":0.629},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73699999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.61599999999999999,"5min":0.75,"15min":0.75},"last":0.48499999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.51800000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.495}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.39100000000000001,"15min":0.39600000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.52700000000000002,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.28599999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.53000000000000003}]},{"osd":1,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.42199999999999999,"15min":0.42499999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57999999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.58199999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.443,"15min":0.44800000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.498},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.437,"15min":0.437},"min":{"1min":0.28199999999999997,"5min":0.252,"15min":0.252},"max":{"1min":0.71599999999999997,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Fri Mar 29 04:31:23 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.58899999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.745},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.72699999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:31:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.58799999999999997,"15min":0.59099999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85399999999999998,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.40999999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.77600000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.63100000000000001,"15min":0.63200000000000001},"min":{"1min":0.47599999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.59999999999999998,"15min":0.59599999999999997},"min":{"1min":0.41699999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.88100000000000001,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.78900000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.76200000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":266163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.499,"15min":0.49199999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.52700000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.41699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.74399999999999999,"5min":0.746,"15min":0.746},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59199999999999997}]},{"osd":2,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52700000000000002,"15min":0.51900000000000002},"min":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.63300000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.45200000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:31:50 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.42099999999999999,"15min":0.40899999999999997},"min":{"1min":0.33000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.45300000000000001,"15min":0.438},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64500000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:31:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.621},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.56499999999999995}]},{"osd":7,"last update":"Fri Mar 29 04:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.66400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":266105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72799999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.69699999999999995,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.623}]},{"osd":1,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56100000000000005},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75800000000000001,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69999999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48899999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41199999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70799999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57499999999999996,"15min":0.57699999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77800000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.50700000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56799999999999995,"15min":0.56599999999999995},"min":{"1min":0.41299999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79900000000000004,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.59599999999999997}]},{"osd":4,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.41999999999999998,"15min":0.41199999999999998},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.42699999999999999,"15min":0.41299999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:32:12 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.44800000000000001,"15min":0.438},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.48699999999999999,"15min":0.47099999999999997},"min":{"1min":0.39800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.65200000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:32:17.014 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:32:17.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:17 smithi029 ceph-mon[30044]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:32:17.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:17 smithi029 ceph-mon[41899]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:32:17.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:17 smithi099 ceph-mon[33865]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:32:17.925 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:18.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:18 smithi099 ceph-mon[33865]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:18.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:18 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:18.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:18 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:18.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:18 smithi029 ceph-mon[30044]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:18.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:18 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:18.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:18 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:18.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:18 smithi029 ceph-mon[41899]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:18.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:18 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:18.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:18 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:20.325 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:32:20.327 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-03-29T04:32:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:20 smithi099 ceph-mon[33865]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:20 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:20.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:20 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:32:20.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:20 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[30044]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[41899]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:32:20.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:20 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:21.147 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-03-29T04:32:19.434933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385548,"kb_used_data":6696,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575668,"statfs":{"total":767960285184,"available":765517484032,"internally_reserved":0,"allocated":6856704,"data_stored":3899686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001820"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548406+0000","last_change":"2024-03-29T04:31:14.925966+0000","last_active":"2024-03-29T04:31:16.548406+0000","last_peered":"2024-03-29T04:31:16.548406+0000","last_clean":"2024-03-29T04:31:16.548406+0000","last_became_active":"2024-03-29T04:31:14.925528+0000","last_became_peered":"2024-03-29T04:31:14.925528+0000","last_unstale":"2024-03-29T04:31:16.548406+0000","last_undegraded":"2024-03-29T04:31:16.548406+0000","last_fullsized":"2024-03-29T04:31:16.548406+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:30:41.616775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548134+0000","last_change":"2024-03-29T04:31:15.188305+0000","last_active":"2024-03-29T04:31:16.548134+0000","last_peered":"2024-03-29T04:31:16.548134+0000","last_clean":"2024-03-29T04:31:16.548134+0000","last_became_active":"2024-03-29T04:31:15.187906+0000","last_became_peered":"2024-03-29T04:31:15.187906+0000","last_unstale":"2024-03-29T04:31:16.548134+0000","last_undegraded":"2024-03-29T04:31:16.548134+0000","last_fullsized":"2024-03-29T04:31:16.548134+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:03:13.618806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924358+0000","last_change":"2024-03-29T04:31:15.188174+0000","last_active":"2024-03-29T04:31:15.924358+0000","last_peered":"2024-03-29T04:31:15.924358+0000","last_clean":"2024-03-29T04:31:15.924358+0000","last_became_active":"2024-03-29T04:31:15.187629+0000","last_became_peered":"2024-03-29T04:31:15.187629+0000","last_unstale":"2024-03-29T04:31:15.924358+0000","last_undegraded":"2024-03-29T04:31:15.924358+0000","last_fullsized":"2024-03-29T04:31:15.924358+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:47:42.088021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924354+0000","last_change":"2024-03-29T04:31:14.926407+0000","last_active":"2024-03-29T04:31:15.924354+0000","last_peered":"2024-03-29T04:31:15.924354+0000","last_clean":"2024-03-29T04:31:15.924354+0000","last_became_active":"2024-03-29T04:31:14.926060+0000","last_became_peered":"2024-03-29T04:31:14.926060+0000","last_unstale":"2024-03-29T04:31:15.924354+0000","last_undegraded":"2024-03-29T04:31:15.924354+0000","last_fullsized":"2024-03-29T04:31:15.924354+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:30.113952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924418+0000","last_change":"2024-03-29T04:30:40.532165+0000","last_active":"2024-03-29T04:31:15.924418+0000","last_peered":"2024-03-29T04:31:15.924418+0000","last_clean":"2024-03-29T04:31:15.924418+0000","last_became_active":"2024-03-29T04:30:40.531726+0000","last_became_peered":"2024-03-29T04:30:40.531726+0000","last_unstale":"2024-03-29T04:31:15.924418+0000","last_undegraded":"2024-03-29T04:31:15.924418+0000","last_fullsized":"2024-03-29T04:31:15.924418+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_clean_scrub_stamp":"2024-03-29T04:28:33.017662+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:56.360284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548399+0000","last_change":"2024-03-29T04:31:14.925369+0000","last_active":"2024-03-29T04:31:16.548399+0000","last_peered":"2024-03-29T04:31:16.548399+0000","last_clean":"2024-03-29T04:31:16.548399+0000","last_became_active":"2024-03-29T04:31:14.924765+0000","last_became_peered":"2024-03-29T04:31:14.924765+0000","last_unstale":"2024-03-29T04:31:16.548399+0000","last_undegraded":"2024-03-29T04:31:16.548399+0000","last_fullsized":"2024-03-29T04:31:16.548399+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:59:12.817095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:15.924014+0000","last_change":"2024-03-29T04:31:14.925939+0000","last_active":"2024-03-29T04:31:15.924014+0000","last_peered":"2024-03-29T04:31:15.924014+0000","last_clean":"2024-03-29T04:31:15.924014+0000","last_became_active":"2024-03-29T04:31:14.925609+0000","last_became_peered":"2024-03-29T04:31:14.925609+0000","last_unstale":"2024-03-29T04:31:15.924014+0000","last_undegraded":"2024-03-29T04:31:15.924014+0000","last_fullsized":"2024-03-29T04:31:15.924014+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:51.860391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548386+0000","last_change":"2024-03-29T04:31:14.924983+0000","last_active":"2024-03-29T04:31:16.548386+0000","last_peered":"2024-03-29T04:31:16.548386+0000","last_clean":"2024-03-29T04:31:16.548386+0000","last_became_active":"2024-03-29T04:31:14.924603+0000","last_became_peered":"2024-03-29T04:31:14.924603+0000","last_unstale":"2024-03-29T04:31:16.548386+0000","last_undegraded":"2024-03-29T04:31:16.548386+0000","last_fullsized":"2024-03-29T04:31:16.548386+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:39:19.333479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:31:16.548153+0000","last_change":"2024-03-29T04:31:14.925317+0000","last_active":"2024-03-29T04:31:16.548153+0000","last_peered":"2024-03-29T04:31:16.548153+0000","last_clean":"2024-03-29T04:31:16.548153+0000","last_became_active":"2024-03-29T04:31:14.925065+0000","last_became_peered":"2024-03-29T04:31:14.925065+0000","last_unstale":"2024-03-29T04:31:16.548153+0000","last_undegraded":"2024-03-29T04:31:16.548153+0000","last_fullsized":"2024-03-29T04:31:16.548153+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:11.187076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.39700000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.52800000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.41199999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.505},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51200000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.375}]},{"osd":5,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.441}]},{"osd":6,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462940,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70299999999999996,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55200000000000005,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.76100000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.67700000000000005}]},{"osd":1,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77600000000000002,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.54400000000000004,"15min":0.54100000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.57499999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.752,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.46800000000000003,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.78300000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.46800000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53100000000000003,"15min":0.53000000000000003},"min":{"1min":0.42899999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79300000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.45600000000000002,"15min":0.45200000000000001},"min":{"1min":0.316,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.71299999999999997,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.46800000000000003,"15min":0.46600000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61799999999999999,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.59399999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.49099999999999999,"15min":0.48099999999999998},"min":{"1min":0.40699999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.63},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.48999999999999999,"15min":0.48399999999999999},"min":{"1min":0.376,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.64500000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:39 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.71599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574900,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:13 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.35499999999999998,"15min":0.33200000000000002},"min":{"1min":0.26800000000000002,"5min":0.224,"15min":0.224},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.36799999999999999,"15min":0.34000000000000002},"min":{"1min":0.252,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.38}]},{"osd":2,"last update":"Fri Mar 29 04:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.28199999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.52000000000000002,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.45000000000000001,"15min":0.441},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.56399999999999995,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42399999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.42599999999999999,"15min":0.439},"min":{"1min":0.27800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.51000000000000001,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.46000000000000002,"15min":0.46899999999999997},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.432}]},{"osd":4,"last update":"Fri Mar 29 04:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70299999999999996,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.52900000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:31:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.499},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77400000000000002,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.63300000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:32:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.55300000000000005,"15min":0.54700000000000004},"min":{"1min":0.41199999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.75700000000000001,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.59799999999999998,"15min":0.59199999999999997},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88800000000000001,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.46800000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:31:41 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738426,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:32:14 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.35799999999999998,"15min":0.33600000000000002},"min":{"1min":0.26200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.39100000000000001,"15min":0.35699999999999998},"min":{"1min":0.28299999999999997,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.61099999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.45500000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:31:39 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.65200000000000002,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40600000000000003,"15min":0.40899999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.53500000000000003,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.61399999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:32:03 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.48199999999999998,"15min":0.48999999999999999},"min":{"1min":0.316,"5min":0.315,"15min":0.315},"max":{"1min":0.53000000000000003,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.46899999999999997,"15min":0.47699999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.54500000000000004,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48899999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.90600000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.68600000000000005,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.64600000000000002,"15min":0.66000000000000003},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.90600000000000003,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.94199999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:32:14 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.626,"15min":0.629},"min":{"1min":0.48899999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80700000000000005,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.59799999999999998,"15min":0.59499999999999997},"min":{"1min":0.42299999999999999,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.89500000000000002,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.89500000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:39 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.85199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.38100000000000001,"15min":0.36899999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.53600000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40400000000000003,"15min":0.38800000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.56699999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.371}]},{"osd":1,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42999999999999999,"15min":0.41999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41899999999999998,"15min":0.41199999999999998},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.58199999999999996,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.253}]},{"osd":3,"last update":"Fri Mar 29 04:32:04 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.45200000000000001,"15min":0.47699999999999998},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.48299999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.51100000000000001,"15min":0.53000000000000003},"min":{"1min":0.32000000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.65200000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.38500000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.45900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.69399999999999995,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.66200000000000003,"15min":0.66500000000000004},"min":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.58799999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:31:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.623,"15min":0.629},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73699999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.61599999999999999,"5min":0.75,"15min":0.75},"last":0.48499999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.51800000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.495}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.39100000000000001,"15min":0.39600000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.52700000000000002,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.28599999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.53000000000000003}]},{"osd":1,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.42199999999999999,"15min":0.42499999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57999999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.58199999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:32:06 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.443,"15min":0.44800000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.498},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.437,"15min":0.437},"min":{"1min":0.28199999999999997,"5min":0.252,"15min":0.252},"max":{"1min":0.71599999999999997,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Fri Mar 29 04:31:23 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58999999999999997,"15min":0.58899999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.745},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.72699999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:31:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.58799999999999997,"15min":0.59099999999999997},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85399999999999998,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.40999999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.77600000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:32:15 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.63100000000000001,"15min":0.63200000000000001},"min":{"1min":0.47599999999999998,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.59999999999999998,"15min":0.59599999999999997},"min":{"1min":0.41699999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.88100000000000001,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.78900000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.76200000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":266163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53000000000000003,"15min":0.52600000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.499,"15min":0.49199999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.52700000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.41699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.74399999999999999,"5min":0.746,"15min":0.746},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59199999999999997}]},{"osd":2,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52700000000000002,"15min":0.51900000000000002},"min":{"1min":0.39400000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.63300000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.53600000000000003,"15min":0.53400000000000003},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.45200000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:31:50 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.42099999999999999,"15min":0.40899999999999997},"min":{"1min":0.33000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.45300000000000001,"15min":0.438},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64500000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:31:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.621},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.56499999999999995}]},{"osd":7,"last update":"Fri Mar 29 04:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.66400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":266105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72799999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.69699999999999995,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.623}]},{"osd":1,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56100000000000005},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75800000000000001,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69999999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48899999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41199999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70799999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57499999999999996,"15min":0.57699999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77800000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.50700000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56799999999999995,"15min":0.56599999999999995},"min":{"1min":0.41299999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79900000000000004,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.59599999999999997}]},{"osd":4,"last update":"Fri Mar 29 04:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.41999999999999998,"15min":0.41199999999999998},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.42699999999999999,"15min":0.41299999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.56699999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:32:12 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.44800000000000001,"15min":0.438},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.48699999999999999,"15min":0.47099999999999997},"min":{"1min":0.39800000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.65200000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:32:21.148 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:32:21.149 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:32:21.149 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:32:21.149 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:32:21.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:21 smithi029 ceph-mon[30044]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:32:21.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:21 smithi029 ceph-mon[30044]: Reconfiguring daemon alertmanager.a on smithi029 2024-03-29T04:32:21.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:21 smithi029 ceph-mon[30044]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:32:21.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:21 smithi029 ceph-mon[41899]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:32:21.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:21 smithi029 ceph-mon[41899]: Reconfiguring daemon alertmanager.a on smithi029 2024-03-29T04:32:21.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:21 smithi029 ceph-mon[41899]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:32:21.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:21 smithi099 ceph-mon[33865]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:32:21.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:21 smithi099 ceph-mon[33865]: Reconfiguring daemon alertmanager.a on smithi029 2024-03-29T04:32:21.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:21 smithi099 ceph-mon[33865]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:32:22.170 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:22.669 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:22 smithi029 ceph-mon[41899]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:22.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:22 smithi029 ceph-mon[30044]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:22.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:22 smithi099 ceph-mon[33865]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:24.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:24 smithi099 ceph-mon[33865]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:24.600 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:24 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:24.600 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:32:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:32:24.919 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:24 smithi029 ceph-mon[41899]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:24.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:24 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:24.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:24 smithi029 ceph-mon[30044]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:24.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:24 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:25.420 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:25 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:32:26.286 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:32:26.286 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-29T04:32:26.286 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[72969]: ts=2024-03-29T04:32:26.190Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:32:26.286 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:26 smithi029 podman[78587]: 2024-03-29 04:32:26.210832319 +0000 UTC m=+0.134689998 container died 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:26.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:26 smithi029 ceph-mon[41899]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:26.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:26 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2530003913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:32:26.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:26 smithi029 ceph-mon[30044]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:26.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:26 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2530003913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:32:26.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:26 smithi099 ceph-mon[33865]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:26.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:26 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2530003913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:32:27.115 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:26 smithi029 podman[78587]: 2024-03-29 04:32:26.742610016 +0000 UTC m=+0.666467691 container cleanup 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:27.115 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:26 smithi029 bash[78587]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:32:27.419 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:27 smithi029 podman[78606]: 2024-03-29 04:32:27.138230944 +0000 UTC m=+0.929800751 container remove 633c38b8a4597e0535452674e96b2d93b72b51d1f0909b6eb7a705c067f3b308 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:27.420 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:27 smithi029 podman[78606]: 2024-03-29 04:32:27.23127096 +0000 UTC m=+1.022840764 volume remove 3382c56fbb7de184134671a14078be08cb24717e0e7e51dc926a2ea2252f1cd9 2024-03-29T04:32:27.637 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-29T04:32:27.637 INFO:tasks.cephadm:Setup complete, yielding 2024-03-29T04:32:27.637 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-29T04:32:27.646 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi029.front.sepia.ceph.com 2024-03-29T04:32:27.646 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- bash -c 'ceph fs volume create foo' 2024-03-29T04:32:27.943 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:28.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:27 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:32:28.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:27 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:32:28.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:27 smithi029 systemd[1]: Starting Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:32:28.465 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:28 smithi029 podman[78817]: 2024-03-29 04:32:28.367107468 +0000 UTC m=+0.096924942 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:32:28.785 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:28 smithi029 ceph-mon[30044]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:28.785 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:28 smithi029 ceph-mon[41899]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:28.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:28 smithi099 ceph-mon[33865]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:29.170 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:28 smithi029 podman[78817]: 2024-03-29 04:32:28.808169272 +0000 UTC m=+0.537986721 volume create ac59ca32c09512f7a0c8567cd603c9ce4963dd23547a540c31f997a9ca96d504 2024-03-29T04:32:30.420 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 podman[78817]: 2024-03-29 04:32:30.131977236 +0000 UTC m=+1.861794688 container create f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:30.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:30 smithi099 ceph-mon[33865]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:30.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:30 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:32:30.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:30 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:32:30.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-mon[41899]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:30.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:32:30.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:32:30.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-mon[30044]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:30.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:32:30.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 podman[78817]: 2024-03-29 04:32:30.462199542 +0000 UTC m=+2.192016997 container init f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 podman[78817]: 2024-03-29 04:32:30.467940837 +0000 UTC m=+2.197758288 container start f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.479Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.479Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.480Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.481Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.504Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.505Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.507Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:30.507Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 bash[78817]: f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 2024-03-29T04:32:30.921 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:30 smithi029 systemd[1]: Started Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:31.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:31 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:31.873 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[30021]: 2024-03-29T04:32:31.571+0000 7f2fdf6bb700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:31.874 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:31 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:32.420 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-29T04:32:32.670 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:32.482Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000533313s 2024-03-29T04:32:32.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:32:32.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: pgmap v49: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: fsmap foo:0 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: Saving service mds.foo spec with placement count:2 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:32.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[30044]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: pgmap v49: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: fsmap foo:0 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: Saving service mds.foo spec with placement count:2 2024-03-29T04:32:32.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:32.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:32 smithi029 ceph-mon[41899]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:32:32.693 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:32.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:32:32.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:32:32.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: pgmap v49: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: fsmap foo:0 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: Saving service mds.foo spec with placement count:2 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:32.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:32 smithi099 ceph-mon[33865]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:32:33.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:33 smithi029 ceph-mon[30044]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:32:33.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:33 smithi029 ceph-mon[41899]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:32:33.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:33 smithi099 ceph-mon[33865]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:32:34.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 systemd[1]: Stopping Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:32:34.561 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:32:34.561 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:32:34.561 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:32:34.561 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:32:34.561 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:32:34.561 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:32:34.561 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[63274]: ts=2024-03-29T04:32:34.390Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:32:34.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 podman[66771]: 2024-03-29 04:32:34.400858488 +0000 UTC m=+0.138448991 container died 7ad2e5eb525c632018ae4760ef7af22a9169359079c1638a5f9c9f6e0e6c4a40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:34.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:34 smithi029 ceph-mon[30044]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:34.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:34 smithi029 ceph-mon[30044]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:32:34.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:34 smithi029 ceph-mon[41899]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:34.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:34 smithi029 ceph-mon[41899]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:32:34.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-mon[33865]: pgmap v53: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:34.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:34 smithi099 ceph-mon[33865]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:32:34.851 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 podman[66771]: 2024-03-29 04:32:34.804018949 +0000 UTC m=+0.541609458 container cleanup 7ad2e5eb525c632018ae4760ef7af22a9169359079c1638a5f9c9f6e0e6c4a40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:34.851 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:34 smithi099 bash[66771]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:32:35.350 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 podman[66790]: 2024-03-29 04:32:35.055445 +0000 UTC m=+0.654424485 container remove 7ad2e5eb525c632018ae4760ef7af22a9169359079c1638a5f9c9f6e0e6c4a40 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:35.350 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:32:35.350 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 systemd[1]: Stopped Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:32:35.350 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.118s CPU time. 2024-03-29T04:32:35.350 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 systemd[1]: Starting Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:32:35.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-29T04:32:35.661 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 podman[66890]: 2024-03-29 04:32:35.39962701 +0000 UTC m=+0.018434520 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 podman[66890]: 2024-03-29 04:32:35.515013948 +0000 UTC m=+0.133821434 container create a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 podman[66890]: 2024-03-29 04:32:35.708988729 +0000 UTC m=+0.327796231 container init a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 podman[66890]: 2024-03-29 04:32:35.71418165 +0000 UTC m=+0.332989136 container start a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.737Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.737Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.737Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi099 (none))" 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.737Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.737Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:32:35.737 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.739Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.740Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.741Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.741Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.743Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.743Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.445µs 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.743Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.743Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.743Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.743Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.055µs wal_replay_duration=378.673µs wbl_replay_duration=113ns total_replay_duration=413.814µs 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.744Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.744Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.744Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:32:35.738 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.759Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.51576ms db_storage=1.069µs remote_storage=1.048µs web_handler=204ns query_engine=569ns scrape=387.441µs scrape_sd=25.962µs notify=23.507µs notify_sd=6.47µs rules=13.664017ms tracing=5.699µs 2024-03-29T04:32:35.739 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.759Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:32:35.739 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:32:35.759Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:32:36.036 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:35] ENGINE Bus STOPPING 2024-03-29T04:32:36.036 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:32:36.036 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:35] ENGINE Bus STOPPED 2024-03-29T04:32:36.037 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:35] ENGINE Bus STARTING 2024-03-29T04:32:36.037 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Serving on http://:::9283 2024-03-29T04:32:36.037 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STARTED 2024-03-29T04:32:36.037 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STOPPING 2024-03-29T04:32:36.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 bash[66890]: a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 2024-03-29T04:32:36.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:32:35 smithi099 systemd[1]: Started Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:32:36.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:32:36.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STOPPED 2024-03-29T04:32:36.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STARTING 2024-03-29T04:32:36.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Serving on http://:::9283 2024-03-29T04:32:36.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STARTED 2024-03-29T04:32:36.420 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STOPPING 2024-03-29T04:32:36.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:32:36.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STOPPED 2024-03-29T04:32:36.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STARTING 2024-03-29T04:32:36.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Serving on http://:::9283 2024-03-29T04:32:36.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:32:36] ENGINE Bus STARTED 2024-03-29T04:32:36.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:36 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:36.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:36.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.904 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:36.905 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:36 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:37.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:37 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:32:37.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:37 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:32:37.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:37 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:32:37.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:37 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:32:37.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:37 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:32:37.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:37 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:32:37.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:32:37.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:32:37.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi029.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:32:37.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:37 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:32:38.416 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-29T04:32:38.429 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:32:38.429 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:32:38.429 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:32:38.668 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:38 smithi029 ceph-mon[41899]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:38.668 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:38 smithi029 ceph-mon[30044]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:38.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:38 smithi099 ceph-mon[33865]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:39.031 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:40.750 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[30044]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:40.750 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[41899]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.751 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.751 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:32:40 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:32:40.483Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002237774s 2024-03-29T04:32:40.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:40 smithi099 ceph-mon[33865]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:40.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:40 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:40 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:40.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:40 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:40.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:40 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:41.263 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:32:41.546 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2651499966' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:32:41.547 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:41 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2651499966' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:32:41.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:41 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2651499966' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:32:42.356 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:42 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:42] "GET /metrics HTTP/1.1" 200 208166 "" "Prometheus/2.43.0" 2024-03-29T04:32:42.386 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":21,"active_gid":24341,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":3374290780},{"type":"v1","addr":"172.21.15.29:6801","nonce":3374290780}]},"active_addr":"172.21.15.29:6801/3374290780","active_change":"2024-03-29T04:31:09.412888+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24367,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1074832730}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":4207961916}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3638347893}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3377829364}]}]} 2024-03-29T04:32:42.390 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:32:42.391 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:32:42.391 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:32:42.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:42 smithi029 ceph-mon[30044]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:42.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:42 smithi029 ceph-mon[41899]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:42.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:42 smithi099 ceph-mon[33865]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:43.137 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:44.409 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: Deploying daemon mds.foo.smithi029.gsovzu on smithi029 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[41899]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:44.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:32:44.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:44.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: Deploying daemon mds.foo.smithi029.gsovzu on smithi029 2024-03-29T04:32:44.411 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:44 smithi029 ceph-mon[30044]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:44.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:32:44.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:44.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: Deploying daemon mds.foo.smithi029.gsovzu on smithi029 2024-03-29T04:32:44.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:44 smithi099 ceph-mon[33865]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:46.133 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:32:46.133 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":68,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:32:32.576509+0000","last_up_change":"2024-03-29T04:30:37.220270+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:31:13.255363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:32:30.488361+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:32:30.810099+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6803","nonce":517562633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6805","nonce":517562633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6809","nonce":517562633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6807","nonce":517562633}]},"public_addr":"172.21.15.29:6803/517562633","cluster_addr":"172.21.15.29:6805/517562633","heartbeat_back_addr":"172.21.15.29:6809/517562633","heartbeat_front_addr":"172.21.15.29:6807/517562633","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6811","nonce":1074269377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6813","nonce":1074269377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6817","nonce":1074269377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6815","nonce":1074269377}]},"public_addr":"172.21.15.29:6811/1074269377","cluster_addr":"172.21.15.29:6813/1074269377","heartbeat_back_addr":"172.21.15.29:6817/1074269377","heartbeat_front_addr":"172.21.15.29:6815/1074269377","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6819","nonce":2548163485}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6821","nonce":2548163485}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6825","nonce":2548163485}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6823","nonce":2548163485}]},"public_addr":"172.21.15.29:6819/2548163485","cluster_addr":"172.21.15.29:6821/2548163485","heartbeat_back_addr":"172.21.15.29:6825/2548163485","heartbeat_front_addr":"172.21.15.29:6823/2548163485","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6827","nonce":2031012271}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6829","nonce":2031012271}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6833","nonce":2031012271}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6831","nonce":2031012271}]},"public_addr":"172.21.15.29:6827/2031012271","cluster_addr":"172.21.15.29:6829/2031012271","heartbeat_back_addr":"172.21.15.29:6833/2031012271","heartbeat_front_addr":"172.21.15.29:6831/2031012271","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:32:46.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:46 smithi029 ceph-mon[30044]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:46.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:46 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/590748613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:32:46.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:46 smithi029 ceph-mon[41899]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:46.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:46 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/590748613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:32:46.809 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:32:46.809 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:32:46.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:46 smithi099 ceph-mon[33865]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:46.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:46 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/590748613' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:32:47.496 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:48.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:48.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:32:48.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:boot 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: daemon mds.foo.smithi029.gsovzu assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: Cluster is now healthy 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: fsmap foo:0 1 up:standby 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:creating} 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: daemon mds.foo.smithi029.gsovzu is now active in filesystem foo as rank 0 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:48.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:48 smithi099 ceph-mon[33865]: Deploying daemon mds.foo.smithi099.jhvkhg on smithi099 2024-03-29T04:32:48.836 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:boot 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: daemon mds.foo.smithi029.gsovzu assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: Cluster is now healthy 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: fsmap foo:0 1 up:standby 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:creating} 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: daemon mds.foo.smithi029.gsovzu is now active in filesystem foo as rank 0 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:48.837 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[30044]: Deploying daemon mds.foo.smithi099.jhvkhg on smithi099 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:boot 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: daemon mds.foo.smithi029.gsovzu assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:32:48.838 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: Cluster is now healthy 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: fsmap foo:0 1 up:standby 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:creating} 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: daemon mds.foo.smithi029.gsovzu is now active in filesystem foo as rank 0 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:32:48.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:48 smithi029 ceph-mon[41899]: Deploying daemon mds.foo.smithi099.jhvkhg on smithi099 2024-03-29T04:32:49.531 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:32:49.531 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":68,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:32:32.576509+0000","last_up_change":"2024-03-29T04:30:37.220270+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:31:13.255363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:32:30.488361+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:32:30.810099+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6803","nonce":517562633}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6805","nonce":517562633}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6809","nonce":517562633}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":517562633},{"type":"v1","addr":"172.21.15.29:6807","nonce":517562633}]},"public_addr":"172.21.15.29:6803/517562633","cluster_addr":"172.21.15.29:6805/517562633","heartbeat_back_addr":"172.21.15.29:6809/517562633","heartbeat_front_addr":"172.21.15.29:6807/517562633","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6811","nonce":1074269377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6813","nonce":1074269377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6817","nonce":1074269377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":1074269377},{"type":"v1","addr":"172.21.15.29:6815","nonce":1074269377}]},"public_addr":"172.21.15.29:6811/1074269377","cluster_addr":"172.21.15.29:6813/1074269377","heartbeat_back_addr":"172.21.15.29:6817/1074269377","heartbeat_front_addr":"172.21.15.29:6815/1074269377","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6819","nonce":2548163485}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6821","nonce":2548163485}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6825","nonce":2548163485}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":2548163485},{"type":"v1","addr":"172.21.15.29:6823","nonce":2548163485}]},"public_addr":"172.21.15.29:6819/2548163485","cluster_addr":"172.21.15.29:6821/2548163485","heartbeat_back_addr":"172.21.15.29:6825/2548163485","heartbeat_front_addr":"172.21.15.29:6823/2548163485","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6827","nonce":2031012271}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6829","nonce":2031012271}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6833","nonce":2031012271}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":2031012271},{"type":"v1","addr":"172.21.15.29:6831","nonce":2031012271}]},"public_addr":"172.21.15.29:6827/2031012271","cluster_addr":"172.21.15.29:6829/2031012271","heartbeat_back_addr":"172.21.15.29:6833/2031012271","heartbeat_front_addr":"172.21.15.29:6831/2031012271","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:32:49.652 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:49 smithi099 ceph-mon[33865]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:active 2024-03-29T04:32:49.652 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:49 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 2024-03-29T04:32:49.848 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:49 smithi029 ceph-mon[30044]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:active 2024-03-29T04:32:49.849 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:49 smithi029 ceph-mon[30044]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 2024-03-29T04:32:49.849 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:49 smithi029 ceph-mon[41899]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:active 2024-03-29T04:32:49.849 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:49 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 2024-03-29T04:32:50.167 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:32:50.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:32:50.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:32:50.168 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:32:50.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:32:50.169 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:32:50.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:32:50.170 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:32:50.765 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.792 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.794 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.799 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.799 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.802 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.803 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.807 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/878835699' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:50 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:50.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/878835699' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/878835699' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:50.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:50 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:32:51.762 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[30044]: mds.? [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] up:boot 2024-03-29T04:32:51.762 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[30044]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:51.762 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:32:51.762 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[30044]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:51.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[41899]: mds.? [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] up:boot 2024-03-29T04:32:51.762 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:51.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:32:51.763 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:51 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:51.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:51 smithi099 ceph-mon[33865]: mds.? [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] up:boot 2024-03-29T04:32:51.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:51 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:51.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:51 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:32:51.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:51 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:52.421 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:52] "GET /metrics HTTP/1.1" 200 208166 "" "Prometheus/2.43.0" 2024-03-29T04:32:52.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:52 smithi099 ceph-mon[33865]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:52.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:52 smithi099 ceph-mon[33865]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:active 2024-03-29T04:32:52.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:52 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:52.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-mon[30044]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:52.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-mon[30044]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:active 2024-03-29T04:32:52.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-mon[30044]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:52.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-mon[41899]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:52.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-mon[41899]: mds.? [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] up:active 2024-03-29T04:32:52.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:52 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:54.601 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:32:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:32:54.601 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-mon[33865]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:54.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:54.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:54 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.774 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[41899]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[30044]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:32:54.775 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:54 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:32:56.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:55 smithi099 ceph-mon[33865]: mds.? [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] up:standby 2024-03-29T04:32:56.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:55 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:56.101 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:55 smithi099 ceph-mon[33865]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:55 smithi029 ceph-mon[30044]: mds.? [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] up:standby 2024-03-29T04:32:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:55 smithi029 ceph-mon[30044]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:56.171 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:55 smithi029 ceph-mon[30044]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:55 smithi029 ceph-mon[41899]: mds.? [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] up:standby 2024-03-29T04:32:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:55 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:32:56.172 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:55 smithi029 ceph-mon[41899]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:58.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:32:58 smithi099 ceph-mon[33865]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:58.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:32:58 smithi029 ceph-mon[30044]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:32:58.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:32:58 smithi029 ceph-mon[41899]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:00.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:00 smithi029 ceph-mon[30044]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:00.671 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:00 smithi029 ceph-mon[41899]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:00.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:00 smithi099 ceph-mon[33865]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:00.858 INFO:teuthology.orchestra.run.smithi029.stdout:111669149746 2024-03-29T04:33:00.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:33:02.378 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:02] "GET /metrics HTTP/1.1" 200 239233 "" "Prometheus/2.43.0" 2024-03-29T04:33:02.671 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:02 smithi029 ceph-mon[30044]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:02.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:02 smithi029 ceph-mon[41899]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:02.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:02 smithi099 ceph-mon[33865]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:33:03.076 INFO:teuthology.orchestra.run.smithi029.stdout:133143986223 2024-03-29T04:33:03.076 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:33:04.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:04 smithi099 ceph-mon[33865]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:33:04.602 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:33:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:33:04.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:04 smithi029 ceph-mon[30044]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:33:04.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:04 smithi029 ceph-mon[41899]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:33:05.010 INFO:teuthology.orchestra.run.smithi029.stdout:236223201312 2024-03-29T04:33:05.011 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:33:05.773 INFO:teuthology.orchestra.run.smithi029.stdout:34359738436 2024-03-29T04:33:05.774 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:33:06.720 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:06 smithi029 ceph-mon[30044]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:33:06.720 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:06 smithi029 ceph-mon[41899]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:33:06.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:06 smithi099 ceph-mon[33865]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:33:08.109 INFO:teuthology.orchestra.run.smithi029.stdout:55834574911 2024-03-29T04:33:08.109 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:33:08.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:08 smithi029 ceph-mon[30044]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:08.672 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:08 smithi029 ceph-mon[41899]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:08.825 INFO:teuthology.orchestra.run.smithi029.stdout:77309411386 2024-03-29T04:33:08.825 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:33:08.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:08 smithi099 ceph-mon[33865]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:09.296 INFO:teuthology.orchestra.run.smithi029.stdout:167503724587 2024-03-29T04:33:09.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:33:09.522 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:09.522 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:09.531 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-03-29T04:33:09.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-03-29T04:33:09.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.532 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:09.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-03-29T04:33:09.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-03-29T04:33:09.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-29T04:33:09.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:10.050 INFO:teuthology.orchestra.run.smithi029.stdout:201863462951 2024-03-29T04:33:10.050 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:33:10.430 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:10.430 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:10.653 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[30044]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:10.653 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-03-29T04:33:10.653 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-03-29T04:33:10.653 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-03-29T04:33:10.653 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[30044]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:33:10.653 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[41899]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:10.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-03-29T04:33:10.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-03-29T04:33:10.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-03-29T04:33:10.654 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:10 smithi029 ceph-mon[41899]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:33:10.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:10 smithi099 ceph-mon[33865]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:10.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:10 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]': finished 2024-03-29T04:33:10.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:10 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-03-29T04:33:10.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:10 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-03-29T04:33:10.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:10 smithi099 ceph-mon[33865]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:33:11.531 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:11.531 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:11.532 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:11.533 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:11.550 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:11 smithi029 ceph-mon[30044]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:33:11.602 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:11 smithi099 ceph-mon[33865]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:33:11.806 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:11 smithi029 ceph-mon[41899]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:33:12.422 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:12] "GET /metrics HTTP/1.1" 200 239198 "" "Prometheus/2.43.0" 2024-03-29T04:33:12.799 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:12 smithi029 ceph-mon[30044]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:12.799 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:12 smithi029 ceph-mon[41899]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:12.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:12 smithi099 ceph-mon[33865]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:14.555 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:33:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:33:14.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:14 smithi099 ceph-mon[33865]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:14.908 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:14 smithi029 ceph-mon[30044]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:14.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:14 smithi029 ceph-mon[41899]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:16.817 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:16 smithi029 ceph-mon[30044]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:16.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:16 smithi029 ceph-mon[41899]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:16.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:16 smithi099 ceph-mon[33865]: pgmap v76: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:16.890 INFO:teuthology.orchestra.run.smithi029.stdout:34359738439 2024-03-29T04:33:17.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:17 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3701879440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:33:17.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:17 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3701879440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:33:17.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:17 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3701879440' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:33:18.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:18 smithi099 ceph-mon[33865]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:18.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:18 smithi029 ceph-mon[30044]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:18.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:18 smithi029 ceph-mon[41899]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:19.117 INFO:teuthology.orchestra.run.smithi029.stdout:111669149751 2024-03-29T04:33:19.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:19 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4035707561' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:33:19.922 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:19 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4035707561' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:33:19.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:19 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4035707561' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:33:20.147 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149751 for osd.3 2024-03-29T04:33:20.147 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:20.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:20 smithi099 ceph-mon[33865]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:20.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:20 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1064352863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:33:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:20 smithi029 ceph-mon[30044]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:20.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:20 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1064352863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:33:20.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:20 smithi029 ceph-mon[41899]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:20.918 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:20 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1064352863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:33:22.366 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738436 got 34359738439 for osd.0 2024-03-29T04:33:22.367 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:22.480 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:22] "GET /metrics HTTP/1.1" 200 239198 "" "Prometheus/2.43.0" 2024-03-29T04:33:22.816 INFO:teuthology.orchestra.run.smithi029.stdout:167503724589 2024-03-29T04:33:22.816 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:22 smithi029 ceph-mon[30044]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:22.817 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:22 smithi029 ceph-mon[41899]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:22.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:22 smithi099 ceph-mon[33865]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:23.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724587 got 167503724589 for osd.5 2024-03-29T04:33:23.693 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:24.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:33:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:33:24.704 INFO:teuthology.orchestra.run.smithi029.stdout:133143986227 2024-03-29T04:33:24.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:24 smithi029 ceph-mon[30044]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:24.839 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:24 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:24.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:24 smithi029 ceph-mon[41899]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:24.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:24 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:24.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:24 smithi099 ceph-mon[33865]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:24.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:24 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:25.148 INFO:teuthology.orchestra.run.smithi029.stdout:236223201316 2024-03-29T04:33:25.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:25 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1347326676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:33:25.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:25 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3116242745' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:33:25.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:25 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/370081920' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:33:25.922 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:25 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1347326676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:33:25.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:25 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3116242745' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:33:25.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:25 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/370081920' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:33:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:25 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1347326676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:33:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:25 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3116242745' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:33:25.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:25 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/370081920' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:33:26.028 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201312 got 236223201316 for osd.7 2024-03-29T04:33:26.028 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:26.420 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986223 got 133143986227 for osd.4 2024-03-29T04:33:26.421 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:26.672 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:26 smithi029 ceph-mon[30044]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:26.673 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:26 smithi029 ceph-mon[41899]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:26.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:26 smithi099 ceph-mon[33865]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:26.999 INFO:teuthology.orchestra.run.smithi029.stdout:77309411389 2024-03-29T04:33:27.797 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411386 got 77309411389 for osd.2 2024-03-29T04:33:27.797 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:28.703 INFO:teuthology.orchestra.run.smithi029.stdout:201863462955 2024-03-29T04:33:28.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:28 smithi099 ceph-mon[33865]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:28.902 INFO:teuthology.orchestra.run.smithi029.stdout:55834574915 2024-03-29T04:33:28.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:28 smithi029 ceph-mon[30044]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:28.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:28 smithi029 ceph-mon[41899]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:29.704 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462951 got 201863462955 for osd.6 2024-03-29T04:33:29.704 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:29.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:29 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/936025201' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:33:29.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:29 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3429589503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:33:29.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:29 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/936025201' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:33:29.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:29 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3429589503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:33:29.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:29 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/936025201' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:33:29.897 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:29 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3429589503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:33:30.224 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574911 got 55834574915 for osd.1 2024-03-29T04:33:30.225 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:30.225 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:33:30.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:33:30.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:30 smithi099 ceph-mon[33865]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:30.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:30 smithi029 ceph-mon[30044]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:30.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:30 smithi029 ceph-mon[41899]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:30.942 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:32.514 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:32] "GET /metrics HTTP/1.1" 200 239232 "" "Prometheus/2.43.0" 2024-03-29T04:33:32.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:32 smithi099 ceph-mon[33865]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:32.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:32 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:32.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:32 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:32.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:32 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.908 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[41899]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[30044]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:32.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:32.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:32.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:32 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:33.690 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:33:33.691 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-03-29T04:33:33.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:33 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:33.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:33 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:33.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:33 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:34.487 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":85,"stamp":"2024-03-29T04:33:33.450862+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386236,"kb_used_data":7368,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574980,"statfs":{"total":767960285184,"available":765516779520,"internally_reserved":0,"allocated":7544832,"data_stored":4254884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002799"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566330+0000","last_change":"2024-03-29T04:32:31.592888+0000","last_active":"2024-03-29T04:33:09.566330+0000","last_peered":"2024-03-29T04:33:09.566330+0000","last_clean":"2024-03-29T04:33:09.566330+0000","last_became_active":"2024-03-29T04:32:31.592647+0000","last_became_peered":"2024-03-29T04:32:31.592647+0000","last_unstale":"2024-03-29T04:33:09.566330+0000","last_undegraded":"2024-03-29T04:33:09.566330+0000","last_fullsized":"2024-03-29T04:33:09.566330+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:49:59.071894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056784+0000","last_change":"2024-03-29T04:32:31.576106+0000","last_active":"2024-03-29T04:33:10.056784+0000","last_peered":"2024-03-29T04:33:10.056784+0000","last_clean":"2024-03-29T04:33:10.056784+0000","last_became_active":"2024-03-29T04:32:31.575248+0000","last_became_peered":"2024-03-29T04:32:31.575248+0000","last_unstale":"2024-03-29T04:33:10.056784+0000","last_undegraded":"2024-03-29T04:33:10.056784+0000","last_fullsized":"2024-03-29T04:33:10.056784+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:48:46.726681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560808+0000","last_change":"2024-03-29T04:32:31.590152+0000","last_active":"2024-03-29T04:33:09.560808+0000","last_peered":"2024-03-29T04:33:09.560808+0000","last_clean":"2024-03-29T04:33:09.560808+0000","last_became_active":"2024-03-29T04:32:31.589845+0000","last_became_peered":"2024-03-29T04:32:31.589845+0000","last_unstale":"2024-03-29T04:33:09.560808+0000","last_undegraded":"2024-03-29T04:33:09.560808+0000","last_fullsized":"2024-03-29T04:33:09.560808+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:38:38.889964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.562164+0000","last_change":"2024-03-29T04:32:31.565434+0000","last_active":"2024-03-29T04:33:09.562164+0000","last_peered":"2024-03-29T04:33:09.562164+0000","last_clean":"2024-03-29T04:33:09.562164+0000","last_became_active":"2024-03-29T04:32:31.562325+0000","last_became_peered":"2024-03-29T04:32:31.562325+0000","last_unstale":"2024-03-29T04:33:09.562164+0000","last_undegraded":"2024-03-29T04:33:09.562164+0000","last_fullsized":"2024-03-29T04:33:09.562164+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:43:20.651381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566580+0000","last_change":"2024-03-29T04:32:31.589669+0000","last_active":"2024-03-29T04:33:09.566580+0000","last_peered":"2024-03-29T04:33:09.566580+0000","last_clean":"2024-03-29T04:33:09.566580+0000","last_became_active":"2024-03-29T04:32:31.589437+0000","last_became_peered":"2024-03-29T04:32:31.589437+0000","last_unstale":"2024-03-29T04:33:09.566580+0000","last_undegraded":"2024-03-29T04:33:09.566580+0000","last_fullsized":"2024-03-29T04:33:09.566580+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:26:27.350025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560922+0000","last_change":"2024-03-29T04:32:31.572333+0000","last_active":"2024-03-29T04:33:09.560922+0000","last_peered":"2024-03-29T04:33:09.560922+0000","last_clean":"2024-03-29T04:33:09.560922+0000","last_became_active":"2024-03-29T04:32:31.572050+0000","last_became_peered":"2024-03-29T04:32:31.572050+0000","last_unstale":"2024-03-29T04:33:09.560922+0000","last_undegraded":"2024-03-29T04:33:09.560922+0000","last_fullsized":"2024-03-29T04:33:09.560922+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:41:46.118602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566716+0000","last_change":"2024-03-29T04:32:31.599296+0000","last_active":"2024-03-29T04:33:09.566716+0000","last_peered":"2024-03-29T04:33:09.566716+0000","last_clean":"2024-03-29T04:33:09.566716+0000","last_became_active":"2024-03-29T04:32:31.598999+0000","last_became_peered":"2024-03-29T04:32:31.598999+0000","last_unstale":"2024-03-29T04:33:09.566716+0000","last_undegraded":"2024-03-29T04:33:09.566716+0000","last_fullsized":"2024-03-29T04:33:09.566716+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:52:11.559051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560733+0000","last_change":"2024-03-29T04:32:31.572619+0000","last_active":"2024-03-29T04:33:09.560733+0000","last_peered":"2024-03-29T04:33:09.560733+0000","last_clean":"2024-03-29T04:33:09.560733+0000","last_became_active":"2024-03-29T04:32:31.572303+0000","last_became_peered":"2024-03-29T04:32:31.572303+0000","last_unstale":"2024-03-29T04:33:09.560733+0000","last_undegraded":"2024-03-29T04:33:09.560733+0000","last_fullsized":"2024-03-29T04:33:09.560733+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:55:51.124870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561128+0000","last_change":"2024-03-29T04:32:31.595348+0000","last_active":"2024-03-29T04:33:09.561128+0000","last_peered":"2024-03-29T04:33:09.561128+0000","last_clean":"2024-03-29T04:33:09.561128+0000","last_became_active":"2024-03-29T04:32:31.595059+0000","last_became_peered":"2024-03-29T04:32:31.595059+0000","last_unstale":"2024-03-29T04:33:09.561128+0000","last_undegraded":"2024-03-29T04:33:09.561128+0000","last_fullsized":"2024-03-29T04:33:09.561128+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:28:08.453900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056649+0000","last_change":"2024-03-29T04:32:31.581209+0000","last_active":"2024-03-29T04:33:10.056649+0000","last_peered":"2024-03-29T04:33:10.056649+0000","last_clean":"2024-03-29T04:33:10.056649+0000","last_became_active":"2024-03-29T04:32:31.580935+0000","last_became_peered":"2024-03-29T04:32:31.580935+0000","last_unstale":"2024-03-29T04:33:10.056649+0000","last_undegraded":"2024-03-29T04:33:10.056649+0000","last_fullsized":"2024-03-29T04:33:10.056649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:18:36.520402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566878+0000","last_change":"2024-03-29T04:32:31.588347+0000","last_active":"2024-03-29T04:33:09.566878+0000","last_peered":"2024-03-29T04:33:09.566878+0000","last_clean":"2024-03-29T04:33:09.566878+0000","last_became_active":"2024-03-29T04:32:31.588080+0000","last_became_peered":"2024-03-29T04:32:31.588080+0000","last_unstale":"2024-03-29T04:33:09.566878+0000","last_undegraded":"2024-03-29T04:33:09.566878+0000","last_fullsized":"2024-03-29T04:33:09.566878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:52:40.449275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056935+0000","last_change":"2024-03-29T04:32:31.578558+0000","last_active":"2024-03-29T04:33:10.056935+0000","last_peered":"2024-03-29T04:33:10.056935+0000","last_clean":"2024-03-29T04:33:10.056935+0000","last_became_active":"2024-03-29T04:32:31.578252+0000","last_became_peered":"2024-03-29T04:32:31.578252+0000","last_unstale":"2024-03-29T04:33:10.056935+0000","last_undegraded":"2024-03-29T04:33:10.056935+0000","last_fullsized":"2024-03-29T04:33:10.056935+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:27:05.696647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568774+0000","last_change":"2024-03-29T04:31:14.925966+0000","last_active":"2024-03-29T04:33:09.568774+0000","last_peered":"2024-03-29T04:33:09.568774+0000","last_clean":"2024-03-29T04:33:09.568774+0000","last_became_active":"2024-03-29T04:31:14.925528+0000","last_became_peered":"2024-03-29T04:31:14.925528+0000","last_unstale":"2024-03-29T04:33:09.568774+0000","last_undegraded":"2024-03-29T04:33:09.568774+0000","last_fullsized":"2024-03-29T04:33:09.568774+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:30:41.616775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560275+0000","last_change":"2024-03-29T04:32:31.772928+0000","last_active":"2024-03-29T04:33:09.560275+0000","last_peered":"2024-03-29T04:33:09.560275+0000","last_clean":"2024-03-29T04:33:09.560275+0000","last_became_active":"2024-03-29T04:32:31.772379+0000","last_became_peered":"2024-03-29T04:32:31.772379+0000","last_unstale":"2024-03-29T04:33:09.560275+0000","last_undegraded":"2024-03-29T04:33:09.560275+0000","last_fullsized":"2024-03-29T04:33:09.560275+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:43:56.698808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567540+0000","last_change":"2024-03-29T04:31:15.188305+0000","last_active":"2024-03-29T04:33:09.567540+0000","last_peered":"2024-03-29T04:33:09.567540+0000","last_clean":"2024-03-29T04:33:09.567540+0000","last_became_active":"2024-03-29T04:31:15.187906+0000","last_became_peered":"2024-03-29T04:31:15.187906+0000","last_unstale":"2024-03-29T04:33:09.567540+0000","last_undegraded":"2024-03-29T04:33:09.567540+0000","last_fullsized":"2024-03-29T04:33:09.567540+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:03:13.618806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.562932+0000","last_change":"2024-03-29T04:32:31.573515+0000","last_active":"2024-03-29T04:33:09.562932+0000","last_peered":"2024-03-29T04:33:09.562932+0000","last_clean":"2024-03-29T04:33:09.562932+0000","last_became_active":"2024-03-29T04:32:31.573229+0000","last_became_peered":"2024-03-29T04:32:31.573229+0000","last_unstale":"2024-03-29T04:33:09.562932+0000","last_undegraded":"2024-03-29T04:33:09.562932+0000","last_fullsized":"2024-03-29T04:33:09.562932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:49:46.508547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056524+0000","last_change":"2024-03-29T04:32:31.772599+0000","last_active":"2024-03-29T04:33:10.056524+0000","last_peered":"2024-03-29T04:33:10.056524+0000","last_clean":"2024-03-29T04:33:10.056524+0000","last_became_active":"2024-03-29T04:32:31.772236+0000","last_became_peered":"2024-03-29T04:32:31.772236+0000","last_unstale":"2024-03-29T04:33:10.056524+0000","last_undegraded":"2024-03-29T04:33:10.056524+0000","last_fullsized":"2024-03-29T04:33:10.056524+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:15:53.964412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568934+0000","last_change":"2024-03-29T04:32:31.573147+0000","last_active":"2024-03-29T04:33:09.568934+0000","last_peered":"2024-03-29T04:33:09.568934+0000","last_clean":"2024-03-29T04:33:09.568934+0000","last_became_active":"2024-03-29T04:32:31.572509+0000","last_became_peered":"2024-03-29T04:32:31.572509+0000","last_unstale":"2024-03-29T04:33:09.568934+0000","last_undegraded":"2024-03-29T04:33:09.568934+0000","last_fullsized":"2024-03-29T04:33:09.568934+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:40:56.358964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565966+0000","last_change":"2024-03-29T04:31:15.188174+0000","last_active":"2024-03-29T04:33:09.565966+0000","last_peered":"2024-03-29T04:33:09.565966+0000","last_clean":"2024-03-29T04:33:09.565966+0000","last_became_active":"2024-03-29T04:31:15.187629+0000","last_became_peered":"2024-03-29T04:31:15.187629+0000","last_unstale":"2024-03-29T04:33:09.565966+0000","last_undegraded":"2024-03-29T04:33:09.565966+0000","last_fullsized":"2024-03-29T04:33:09.565966+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:47:42.088021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.057001+0000","last_change":"2024-03-29T04:32:31.595358+0000","last_active":"2024-03-29T04:33:10.057001+0000","last_peered":"2024-03-29T04:33:10.057001+0000","last_clean":"2024-03-29T04:33:10.057001+0000","last_became_active":"2024-03-29T04:32:31.595051+0000","last_became_peered":"2024-03-29T04:32:31.595051+0000","last_unstale":"2024-03-29T04:33:10.057001+0000","last_undegraded":"2024-03-29T04:33:10.057001+0000","last_fullsized":"2024-03-29T04:33:10.057001+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:15:09.145501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056407+0000","last_change":"2024-03-29T04:32:31.574916+0000","last_active":"2024-03-29T04:33:10.056407+0000","last_peered":"2024-03-29T04:33:10.056407+0000","last_clean":"2024-03-29T04:33:10.056407+0000","last_became_active":"2024-03-29T04:32:31.574627+0000","last_became_peered":"2024-03-29T04:32:31.574627+0000","last_unstale":"2024-03-29T04:33:10.056407+0000","last_undegraded":"2024-03-29T04:33:10.056407+0000","last_fullsized":"2024-03-29T04:33:10.056407+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:45:31.424895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565826+0000","last_change":"2024-03-29T04:31:14.926407+0000","last_active":"2024-03-29T04:33:09.565826+0000","last_peered":"2024-03-29T04:33:09.565826+0000","last_clean":"2024-03-29T04:33:09.565826+0000","last_became_active":"2024-03-29T04:31:14.926060+0000","last_became_peered":"2024-03-29T04:31:14.926060+0000","last_unstale":"2024-03-29T04:33:09.565826+0000","last_undegraded":"2024-03-29T04:33:09.565826+0000","last_fullsized":"2024-03-29T04:33:09.565826+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:30.113952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568623+0000","last_change":"2024-03-29T04:32:31.772849+0000","last_active":"2024-03-29T04:33:09.568623+0000","last_peered":"2024-03-29T04:33:09.568623+0000","last_clean":"2024-03-29T04:33:09.568623+0000","last_became_active":"2024-03-29T04:32:31.772561+0000","last_became_peered":"2024-03-29T04:32:31.772561+0000","last_unstale":"2024-03-29T04:33:09.568623+0000","last_undegraded":"2024-03-29T04:33:09.568623+0000","last_fullsized":"2024-03-29T04:33:09.568623+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:31.653522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561123+0000","last_change":"2024-03-29T04:32:31.567152+0000","last_active":"2024-03-29T04:33:09.561123+0000","last_peered":"2024-03-29T04:33:09.561123+0000","last_clean":"2024-03-29T04:33:09.561123+0000","last_became_active":"2024-03-29T04:32:31.566871+0000","last_became_peered":"2024-03-29T04:32:31.566871+0000","last_unstale":"2024-03-29T04:33:09.561123+0000","last_undegraded":"2024-03-29T04:33:09.561123+0000","last_fullsized":"2024-03-29T04:33:09.561123+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:44:25.854105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565705+0000","last_change":"2024-03-29T04:30:40.532165+0000","last_active":"2024-03-29T04:33:09.565705+0000","last_peered":"2024-03-29T04:33:09.565705+0000","last_clean":"2024-03-29T04:33:09.565705+0000","last_became_active":"2024-03-29T04:30:40.531726+0000","last_became_peered":"2024-03-29T04:30:40.531726+0000","last_unstale":"2024-03-29T04:33:09.565705+0000","last_undegraded":"2024-03-29T04:33:09.565705+0000","last_fullsized":"2024-03-29T04:33:09.565705+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_clean_scrub_stamp":"2024-03-29T04:28:33.017662+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:56.360284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560684+0000","last_change":"2024-03-29T04:31:14.925369+0000","last_active":"2024-03-29T04:33:09.560684+0000","last_peered":"2024-03-29T04:33:09.560684+0000","last_clean":"2024-03-29T04:33:09.560684+0000","last_became_active":"2024-03-29T04:31:14.924765+0000","last_became_peered":"2024-03-29T04:31:14.924765+0000","last_unstale":"2024-03-29T04:33:09.560684+0000","last_undegraded":"2024-03-29T04:33:09.560684+0000","last_fullsized":"2024-03-29T04:33:09.560684+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:59:12.817095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567674+0000","last_change":"2024-03-29T04:32:31.597573+0000","last_active":"2024-03-29T04:33:09.567674+0000","last_peered":"2024-03-29T04:33:09.567674+0000","last_clean":"2024-03-29T04:33:09.567674+0000","last_became_active":"2024-03-29T04:32:31.596766+0000","last_became_peered":"2024-03-29T04:32:31.596766+0000","last_unstale":"2024-03-29T04:33:09.567674+0000","last_undegraded":"2024-03-29T04:33:09.567674+0000","last_fullsized":"2024-03-29T04:33:09.567674+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:47:45.530721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567140+0000","last_change":"2024-03-29T04:32:31.567031+0000","last_active":"2024-03-29T04:33:09.567140+0000","last_peered":"2024-03-29T04:33:09.567140+0000","last_clean":"2024-03-29T04:33:09.567140+0000","last_became_active":"2024-03-29T04:32:31.566719+0000","last_became_peered":"2024-03-29T04:32:31.566719+0000","last_unstale":"2024-03-29T04:33:09.567140+0000","last_undegraded":"2024-03-29T04:33:09.567140+0000","last_fullsized":"2024-03-29T04:33:09.567140+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:17:38.096048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561995+0000","last_change":"2024-03-29T04:31:14.925939+0000","last_active":"2024-03-29T04:33:09.561995+0000","last_peered":"2024-03-29T04:33:09.561995+0000","last_clean":"2024-03-29T04:33:09.561995+0000","last_became_active":"2024-03-29T04:31:14.925609+0000","last_became_peered":"2024-03-29T04:31:14.925609+0000","last_unstale":"2024-03-29T04:33:09.561995+0000","last_undegraded":"2024-03-29T04:33:09.561995+0000","last_fullsized":"2024-03-29T04:33:09.561995+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:51.860391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568405+0000","last_change":"2024-03-29T04:32:31.598332+0000","last_active":"2024-03-29T04:33:09.568405+0000","last_peered":"2024-03-29T04:33:09.568405+0000","last_clean":"2024-03-29T04:33:09.568405+0000","last_became_active":"2024-03-29T04:32:31.597204+0000","last_became_peered":"2024-03-29T04:32:31.597204+0000","last_unstale":"2024-03-29T04:33:09.568405+0000","last_undegraded":"2024-03-29T04:33:09.568405+0000","last_fullsized":"2024-03-29T04:33:09.568405+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:46:05.313560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566621+0000","last_change":"2024-03-29T04:32:31.574292+0000","last_active":"2024-03-29T04:33:09.566621+0000","last_peered":"2024-03-29T04:33:09.566621+0000","last_clean":"2024-03-29T04:33:09.566621+0000","last_became_active":"2024-03-29T04:32:31.573679+0000","last_became_peered":"2024-03-29T04:32:31.573679+0000","last_unstale":"2024-03-29T04:33:09.566621+0000","last_undegraded":"2024-03-29T04:33:09.566621+0000","last_fullsized":"2024-03-29T04:33:09.566621+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:51:01.779754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560612+0000","last_change":"2024-03-29T04:31:14.924983+0000","last_active":"2024-03-29T04:33:09.560612+0000","last_peered":"2024-03-29T04:33:09.560612+0000","last_clean":"2024-03-29T04:33:09.560612+0000","last_became_active":"2024-03-29T04:31:14.924603+0000","last_became_peered":"2024-03-29T04:31:14.924603+0000","last_unstale":"2024-03-29T04:33:09.560612+0000","last_undegraded":"2024-03-29T04:33:09.560612+0000","last_fullsized":"2024-03-29T04:33:09.560612+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:39:19.333479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567398+0000","last_change":"2024-03-29T04:32:31.599616+0000","last_active":"2024-03-29T04:33:09.567398+0000","last_peered":"2024-03-29T04:33:09.567398+0000","last_clean":"2024-03-29T04:33:09.567398+0000","last_became_active":"2024-03-29T04:32:31.599131+0000","last_became_peered":"2024-03-29T04:32:31.599131+0000","last_unstale":"2024-03-29T04:33:09.567398+0000","last_undegraded":"2024-03-29T04:33:09.567398+0000","last_fullsized":"2024-03-29T04:33:09.567398+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:09:17.407227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560907+0000","last_change":"2024-03-29T04:32:31.576596+0000","last_active":"2024-03-29T04:33:09.560907+0000","last_peered":"2024-03-29T04:33:09.560907+0000","last_clean":"2024-03-29T04:33:09.560907+0000","last_became_active":"2024-03-29T04:32:31.576298+0000","last_became_peered":"2024-03-29T04:32:31.576298+0000","last_unstale":"2024-03-29T04:33:09.560907+0000","last_undegraded":"2024-03-29T04:33:09.560907+0000","last_fullsized":"2024-03-29T04:33:09.560907+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:19:56.607906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568324+0000","last_change":"2024-03-29T04:31:14.925317+0000","last_active":"2024-03-29T04:33:09.568324+0000","last_peered":"2024-03-29T04:33:09.568324+0000","last_clean":"2024-03-29T04:33:09.568324+0000","last_became_active":"2024-03-29T04:31:14.925065+0000","last_became_peered":"2024-03-29T04:31:14.925065+0000","last_unstale":"2024-03-29T04:33:09.568324+0000","last_undegraded":"2024-03-29T04:33:09.568324+0000","last_fullsized":"2024-03-29T04:33:09.568324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:11.187076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:33:11.002270+0000","last_change":"2024-03-29T04:33:11.002270+0000","last_active":"2024-03-29T04:33:11.002270+0000","last_peered":"2024-03-29T04:33:11.002270+0000","last_clean":"2024-03-29T04:33:11.002270+0000","last_became_active":"2024-03-29T04:33:11.001967+0000","last_became_peered":"2024-03-29T04:33:11.001967+0000","last_unstale":"2024-03-29T04:33:11.002270+0000","last_undegraded":"2024-03-29T04:33:11.002270+0000","last_fullsized":"2024-03-29T04:33:11.002270+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:45:46.932749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560414+0000","last_change":"2024-03-29T04:32:31.571365+0000","last_active":"2024-03-29T04:33:09.560414+0000","last_peered":"2024-03-29T04:33:09.560414+0000","last_clean":"2024-03-29T04:33:09.560414+0000","last_became_active":"2024-03-29T04:32:31.570932+0000","last_became_peered":"2024-03-29T04:32:31.570932+0000","last_unstale":"2024-03-29T04:33:09.560414+0000","last_undegraded":"2024-03-29T04:33:09.560414+0000","last_fullsized":"2024-03-29T04:33:09.560414+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:17:21.103008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567154+0000","last_change":"2024-03-29T04:32:31.593605+0000","last_active":"2024-03-29T04:33:09.567154+0000","last_peered":"2024-03-29T04:33:09.567154+0000","last_clean":"2024-03-29T04:33:09.567154+0000","last_became_active":"2024-03-29T04:32:31.593330+0000","last_became_peered":"2024-03-29T04:32:31.593330+0000","last_unstale":"2024-03-29T04:33:09.567154+0000","last_undegraded":"2024-03-29T04:33:09.567154+0000","last_fullsized":"2024-03-29T04:33:09.567154+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:40.015461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566969+0000","last_change":"2024-03-29T04:32:31.568721+0000","last_active":"2024-03-29T04:33:09.566969+0000","last_peered":"2024-03-29T04:33:09.566969+0000","last_clean":"2024-03-29T04:33:09.566969+0000","last_became_active":"2024-03-29T04:32:31.568416+0000","last_became_peered":"2024-03-29T04:32:31.568416+0000","last_unstale":"2024-03-29T04:33:09.566969+0000","last_undegraded":"2024-03-29T04:33:09.566969+0000","last_fullsized":"2024-03-29T04:33:09.566969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:15:38.580168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567250+0000","last_change":"2024-03-29T04:32:31.593457+0000","last_active":"2024-03-29T04:33:09.567250+0000","last_peered":"2024-03-29T04:33:09.567250+0000","last_clean":"2024-03-29T04:33:09.567250+0000","last_became_active":"2024-03-29T04:32:31.593199+0000","last_became_peered":"2024-03-29T04:32:31.593199+0000","last_unstale":"2024-03-29T04:33:09.567250+0000","last_undegraded":"2024-03-29T04:33:09.567250+0000","last_fullsized":"2024-03-29T04:33:09.567250+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:08:56.455967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568562+0000","last_change":"2024-03-29T04:32:31.574602+0000","last_active":"2024-03-29T04:33:09.568562+0000","last_peered":"2024-03-29T04:33:09.568562+0000","last_clean":"2024-03-29T04:33:09.568562+0000","last_became_active":"2024-03-29T04:32:31.574341+0000","last_became_peered":"2024-03-29T04:32:31.574341+0000","last_unstale":"2024-03-29T04:33:09.568562+0000","last_undegraded":"2024-03-29T04:33:09.568562+0000","last_fullsized":"2024-03-29T04:33:09.568562+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:41:48.123074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567005+0000","last_change":"2024-03-29T04:32:31.589205+0000","last_active":"2024-03-29T04:33:09.567005+0000","last_peered":"2024-03-29T04:33:09.567005+0000","last_clean":"2024-03-29T04:33:09.567005+0000","last_became_active":"2024-03-29T04:32:31.588852+0000","last_became_peered":"2024-03-29T04:32:31.588852+0000","last_unstale":"2024-03-29T04:33:09.567005+0000","last_undegraded":"2024-03-29T04:33:09.567005+0000","last_fullsized":"2024-03-29T04:33:09.567005+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:02:51.103250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561274+0000","last_change":"2024-03-29T04:32:31.573299+0000","last_active":"2024-03-29T04:33:09.561274+0000","last_peered":"2024-03-29T04:33:09.561274+0000","last_clean":"2024-03-29T04:33:09.561274+0000","last_became_active":"2024-03-29T04:32:31.572968+0000","last_became_peered":"2024-03-29T04:32:31.572968+0000","last_unstale":"2024-03-29T04:33:09.561274+0000","last_undegraded":"2024-03-29T04:33:09.561274+0000","last_fullsized":"2024-03-29T04:33:09.561274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:31:45.450843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056665+0000","last_change":"2024-03-29T04:32:31.595919+0000","last_active":"2024-03-29T04:33:10.056665+0000","last_peered":"2024-03-29T04:33:10.056665+0000","last_clean":"2024-03-29T04:33:10.056665+0000","last_became_active":"2024-03-29T04:32:31.595623+0000","last_became_peered":"2024-03-29T04:32:31.595623+0000","last_unstale":"2024-03-29T04:33:10.056665+0000","last_undegraded":"2024-03-29T04:33:10.056665+0000","last_fullsized":"2024-03-29T04:33:10.056665+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:26:11.492507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560820+0000","last_change":"2024-03-29T04:32:31.575921+0000","last_active":"2024-03-29T04:33:09.560820+0000","last_peered":"2024-03-29T04:33:09.560820+0000","last_clean":"2024-03-29T04:33:09.560820+0000","last_became_active":"2024-03-29T04:32:31.575668+0000","last_became_peered":"2024-03-29T04:32:31.575668+0000","last_unstale":"2024-03-29T04:33:09.560820+0000","last_undegraded":"2024-03-29T04:33:09.560820+0000","last_fullsized":"2024-03-29T04:33:09.560820+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:07:35.477597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.720781+0000","last_change":"2024-03-29T04:33:10.720781+0000","last_active":"2024-03-29T04:33:10.720781+0000","last_peered":"2024-03-29T04:33:10.720781+0000","last_clean":"2024-03-29T04:33:10.720781+0000","last_became_active":"2024-03-29T04:33:10.720378+0000","last_became_peered":"2024-03-29T04:33:10.720378+0000","last_unstale":"2024-03-29T04:33:10.720781+0000","last_undegraded":"2024-03-29T04:33:10.720781+0000","last_fullsized":"2024-03-29T04:33:10.720781+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:20:45.090317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566125+0000","last_change":"2024-03-29T04:32:31.573699+0000","last_active":"2024-03-29T04:33:09.566125+0000","last_peered":"2024-03-29T04:33:09.566125+0000","last_clean":"2024-03-29T04:33:09.566125+0000","last_became_active":"2024-03-29T04:32:31.573265+0000","last_became_peered":"2024-03-29T04:32:31.573265+0000","last_unstale":"2024-03-29T04:33:09.566125+0000","last_undegraded":"2024-03-29T04:33:09.566125+0000","last_fullsized":"2024-03-29T04:33:09.566125+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:05:16.290933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566678+0000","last_change":"2024-03-29T04:32:31.593225+0000","last_active":"2024-03-29T04:33:09.566678+0000","last_peered":"2024-03-29T04:33:09.566678+0000","last_clean":"2024-03-29T04:33:09.566678+0000","last_became_active":"2024-03-29T04:32:31.592627+0000","last_became_peered":"2024-03-29T04:32:31.592627+0000","last_unstale":"2024-03-29T04:33:09.566678+0000","last_undegraded":"2024-03-29T04:33:09.566678+0000","last_fullsized":"2024-03-29T04:33:09.566678+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:47:14.028885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566024+0000","last_change":"2024-03-29T04:32:31.586692+0000","last_active":"2024-03-29T04:33:09.566024+0000","last_peered":"2024-03-29T04:33:09.566024+0000","last_clean":"2024-03-29T04:33:09.566024+0000","last_became_active":"2024-03-29T04:32:31.586390+0000","last_became_peered":"2024-03-29T04:32:31.586390+0000","last_unstale":"2024-03-29T04:33:09.566024+0000","last_undegraded":"2024-03-29T04:33:09.566024+0000","last_fullsized":"2024-03-29T04:33:09.566024+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:14:19.075337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560569+0000","last_change":"2024-03-29T04:32:31.772869+0000","last_active":"2024-03-29T04:33:09.560569+0000","last_peered":"2024-03-29T04:33:09.560569+0000","last_clean":"2024-03-29T04:33:09.560569+0000","last_became_active":"2024-03-29T04:32:31.772518+0000","last_became_peered":"2024-03-29T04:32:31.772518+0000","last_unstale":"2024-03-29T04:33:09.560569+0000","last_undegraded":"2024-03-29T04:33:09.560569+0000","last_fullsized":"2024-03-29T04:33:09.560569+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:03:29.625832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566052+0000","last_change":"2024-03-29T04:32:31.580267+0000","last_active":"2024-03-29T04:33:09.566052+0000","last_peered":"2024-03-29T04:33:09.566052+0000","last_clean":"2024-03-29T04:33:09.566052+0000","last_became_active":"2024-03-29T04:32:31.580016+0000","last_became_peered":"2024-03-29T04:32:31.580016+0000","last_unstale":"2024-03-29T04:33:09.566052+0000","last_undegraded":"2024-03-29T04:33:09.566052+0000","last_fullsized":"2024-03-29T04:33:09.566052+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:35:44.602037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.563012+0000","last_change":"2024-03-29T04:32:31.772688+0000","last_active":"2024-03-29T04:33:09.563012+0000","last_peered":"2024-03-29T04:33:09.563012+0000","last_clean":"2024-03-29T04:33:09.563012+0000","last_became_active":"2024-03-29T04:32:31.772378+0000","last_became_peered":"2024-03-29T04:32:31.772378+0000","last_unstale":"2024-03-29T04:33:09.563012+0000","last_undegraded":"2024-03-29T04:33:09.563012+0000","last_fullsized":"2024-03-29T04:33:09.563012+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:41:34.928801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568471+0000","last_change":"2024-03-29T04:32:31.570678+0000","last_active":"2024-03-29T04:33:09.568471+0000","last_peered":"2024-03-29T04:33:09.568471+0000","last_clean":"2024-03-29T04:33:09.568471+0000","last_became_active":"2024-03-29T04:32:31.570015+0000","last_became_peered":"2024-03-29T04:32:31.570015+0000","last_unstale":"2024-03-29T04:33:09.568471+0000","last_undegraded":"2024-03-29T04:33:09.568471+0000","last_fullsized":"2024-03-29T04:33:09.568471+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:19:55.110568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056849+0000","last_change":"2024-03-29T04:32:31.773532+0000","last_active":"2024-03-29T04:33:10.056849+0000","last_peered":"2024-03-29T04:33:10.056849+0000","last_clean":"2024-03-29T04:33:10.056849+0000","last_became_active":"2024-03-29T04:32:31.773256+0000","last_became_peered":"2024-03-29T04:32:31.773256+0000","last_unstale":"2024-03-29T04:33:10.056849+0000","last_undegraded":"2024-03-29T04:33:10.056849+0000","last_fullsized":"2024-03-29T04:33:10.056849+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:46:29.922937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565650+0000","last_change":"2024-03-29T04:32:31.578200+0000","last_active":"2024-03-29T04:33:09.565650+0000","last_peered":"2024-03-29T04:33:09.565650+0000","last_clean":"2024-03-29T04:33:09.565650+0000","last_became_active":"2024-03-29T04:32:31.577900+0000","last_became_peered":"2024-03-29T04:32:31.577900+0000","last_unstale":"2024-03-29T04:33:09.565650+0000","last_undegraded":"2024-03-29T04:33:09.565650+0000","last_fullsized":"2024-03-29T04:33:09.565650+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:04:23.132474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560788+0000","last_change":"2024-03-29T04:32:31.774941+0000","last_active":"2024-03-29T04:33:09.560788+0000","last_peered":"2024-03-29T04:33:09.560788+0000","last_clean":"2024-03-29T04:33:09.560788+0000","last_became_active":"2024-03-29T04:32:31.774665+0000","last_became_peered":"2024-03-29T04:32:31.774665+0000","last_unstale":"2024-03-29T04:33:09.560788+0000","last_undegraded":"2024-03-29T04:33:09.560788+0000","last_fullsized":"2024-03-29T04:33:09.560788+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:16:08.260190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056529+0000","last_change":"2024-03-29T04:32:31.577802+0000","last_active":"2024-03-29T04:33:10.056529+0000","last_peered":"2024-03-29T04:33:10.056529+0000","last_clean":"2024-03-29T04:33:10.056529+0000","last_became_active":"2024-03-29T04:32:31.577475+0000","last_became_peered":"2024-03-29T04:32:31.577475+0000","last_unstale":"2024-03-29T04:33:10.056529+0000","last_undegraded":"2024-03-29T04:33:10.056529+0000","last_fullsized":"2024-03-29T04:33:10.056529+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:25:09.560054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.720797+0000","last_change":"2024-03-29T04:33:10.720797+0000","last_active":"2024-03-29T04:33:10.720797+0000","last_peered":"2024-03-29T04:33:10.720797+0000","last_clean":"2024-03-29T04:33:10.720797+0000","last_became_active":"2024-03-29T04:33:10.720516+0000","last_became_peered":"2024-03-29T04:33:10.720516+0000","last_unstale":"2024-03-29T04:33:10.720797+0000","last_undegraded":"2024-03-29T04:33:10.720797+0000","last_fullsized":"2024-03-29T04:33:10.720797+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:37:11.811412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566175+0000","last_change":"2024-03-29T04:32:31.569884+0000","last_active":"2024-03-29T04:33:09.566175+0000","last_peered":"2024-03-29T04:33:09.566175+0000","last_clean":"2024-03-29T04:33:09.566175+0000","last_became_active":"2024-03-29T04:32:31.569682+0000","last_became_peered":"2024-03-29T04:32:31.569682+0000","last_unstale":"2024-03-29T04:33:09.566175+0000","last_undegraded":"2024-03-29T04:33:09.566175+0000","last_fullsized":"2024-03-29T04:33:09.566175+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:49:57.154622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566643+0000","last_change":"2024-03-29T04:32:31.599699+0000","last_active":"2024-03-29T04:33:09.566643+0000","last_peered":"2024-03-29T04:33:09.566643+0000","last_clean":"2024-03-29T04:33:09.566643+0000","last_became_active":"2024-03-29T04:32:31.599298+0000","last_became_peered":"2024-03-29T04:32:31.599298+0000","last_unstale":"2024-03-29T04:33:09.566643+0000","last_undegraded":"2024-03-29T04:33:09.566643+0000","last_fullsized":"2024-03-29T04:33:09.566643+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:16:52.575247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566907+0000","last_change":"2024-03-29T04:32:31.567585+0000","last_active":"2024-03-29T04:33:09.566907+0000","last_peered":"2024-03-29T04:33:09.566907+0000","last_clean":"2024-03-29T04:33:09.566907+0000","last_became_active":"2024-03-29T04:32:31.567306+0000","last_became_peered":"2024-03-29T04:32:31.567306+0000","last_unstale":"2024-03-29T04:33:09.566907+0000","last_undegraded":"2024-03-29T04:33:09.566907+0000","last_fullsized":"2024-03-29T04:33:09.566907+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:48:36.533604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567181+0000","last_change":"2024-03-29T04:32:31.597871+0000","last_active":"2024-03-29T04:33:09.567181+0000","last_peered":"2024-03-29T04:33:09.567181+0000","last_clean":"2024-03-29T04:33:09.567181+0000","last_became_active":"2024-03-29T04:32:31.597588+0000","last_became_peered":"2024-03-29T04:32:31.597588+0000","last_unstale":"2024-03-29T04:33:09.567181+0000","last_undegraded":"2024-03-29T04:33:09.567181+0000","last_fullsized":"2024-03-29T04:33:09.567181+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:56:24.378566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566029+0000","last_change":"2024-03-29T04:32:31.582734+0000","last_active":"2024-03-29T04:33:09.566029+0000","last_peered":"2024-03-29T04:33:09.566029+0000","last_clean":"2024-03-29T04:33:09.566029+0000","last_became_active":"2024-03-29T04:32:31.581340+0000","last_became_peered":"2024-03-29T04:32:31.581340+0000","last_unstale":"2024-03-29T04:33:09.566029+0000","last_undegraded":"2024-03-29T04:33:09.566029+0000","last_fullsized":"2024-03-29T04:33:09.566029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:26:54.916930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560878+0000","last_change":"2024-03-29T04:32:31.773690+0000","last_active":"2024-03-29T04:33:09.560878+0000","last_peered":"2024-03-29T04:33:09.560878+0000","last_clean":"2024-03-29T04:33:09.560878+0000","last_became_active":"2024-03-29T04:32:31.773395+0000","last_became_peered":"2024-03-29T04:32:31.773395+0000","last_unstale":"2024-03-29T04:33:09.560878+0000","last_undegraded":"2024-03-29T04:33:09.560878+0000","last_fullsized":"2024-03-29T04:33:09.560878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:12:59.816689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560863+0000","last_change":"2024-03-29T04:32:31.572581+0000","last_active":"2024-03-29T04:33:09.560863+0000","last_peered":"2024-03-29T04:33:09.560863+0000","last_clean":"2024-03-29T04:33:09.560863+0000","last_became_active":"2024-03-29T04:32:31.572271+0000","last_became_peered":"2024-03-29T04:32:31.572271+0000","last_unstale":"2024-03-29T04:33:09.560863+0000","last_undegraded":"2024-03-29T04:33:09.560863+0000","last_fullsized":"2024-03-29T04:33:09.560863+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:50:10.131117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560417+0000","last_change":"2024-03-29T04:32:31.593572+0000","last_active":"2024-03-29T04:33:09.560417+0000","last_peered":"2024-03-29T04:33:09.560417+0000","last_clean":"2024-03-29T04:33:09.560417+0000","last_became_active":"2024-03-29T04:32:31.593256+0000","last_became_peered":"2024-03-29T04:32:31.593256+0000","last_unstale":"2024-03-29T04:33:09.560417+0000","last_undegraded":"2024-03-29T04:33:09.560417+0000","last_fullsized":"2024-03-29T04:33:09.560417+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:16:03.370981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056692+0000","last_change":"2024-03-29T04:32:31.577919+0000","last_active":"2024-03-29T04:33:10.056692+0000","last_peered":"2024-03-29T04:33:10.056692+0000","last_clean":"2024-03-29T04:33:10.056692+0000","last_became_active":"2024-03-29T04:32:31.577490+0000","last_became_peered":"2024-03-29T04:32:31.577490+0000","last_unstale":"2024-03-29T04:33:10.056692+0000","last_undegraded":"2024-03-29T04:33:10.056692+0000","last_fullsized":"2024-03-29T04:33:10.056692+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:32:51.541477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568722+0000","last_change":"2024-03-29T04:32:31.599080+0000","last_active":"2024-03-29T04:33:09.568722+0000","last_peered":"2024-03-29T04:33:09.568722+0000","last_clean":"2024-03-29T04:33:09.568722+0000","last_became_active":"2024-03-29T04:32:31.598739+0000","last_became_peered":"2024-03-29T04:32:31.598739+0000","last_unstale":"2024-03-29T04:33:09.568722+0000","last_undegraded":"2024-03-29T04:33:09.568722+0000","last_fullsized":"2024-03-29T04:33:09.568722+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:08:32.915193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561872+0000","last_change":"2024-03-29T04:32:31.572300+0000","last_active":"2024-03-29T04:33:09.561872+0000","last_peered":"2024-03-29T04:33:09.561872+0000","last_clean":"2024-03-29T04:33:09.561872+0000","last_became_active":"2024-03-29T04:32:31.571969+0000","last_became_peered":"2024-03-29T04:32:31.571969+0000","last_unstale":"2024-03-29T04:33:09.561872+0000","last_undegraded":"2024-03-29T04:33:09.561872+0000","last_fullsized":"2024-03-29T04:33:09.561872+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:34:49.039328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056925+0000","last_change":"2024-03-29T04:32:31.773604+0000","last_active":"2024-03-29T04:33:10.056925+0000","last_peered":"2024-03-29T04:33:10.056925+0000","last_clean":"2024-03-29T04:33:10.056925+0000","last_became_active":"2024-03-29T04:32:31.773333+0000","last_became_peered":"2024-03-29T04:32:31.773333+0000","last_unstale":"2024-03-29T04:33:10.056925+0000","last_undegraded":"2024-03-29T04:33:10.056925+0000","last_fullsized":"2024-03-29T04:33:10.056925+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:19:59.776976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567613+0000","last_change":"2024-03-29T04:32:31.572883+0000","last_active":"2024-03-29T04:33:09.567613+0000","last_peered":"2024-03-29T04:33:09.567613+0000","last_clean":"2024-03-29T04:33:09.567613+0000","last_became_active":"2024-03-29T04:32:31.572588+0000","last_became_peered":"2024-03-29T04:32:31.572588+0000","last_unstale":"2024-03-29T04:33:09.567613+0000","last_undegraded":"2024-03-29T04:33:09.567613+0000","last_fullsized":"2024-03-29T04:33:09.567613+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:48:51.961446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.563219+0000","last_change":"2024-03-29T04:32:31.594266+0000","last_active":"2024-03-29T04:33:09.563219+0000","last_peered":"2024-03-29T04:33:09.563219+0000","last_clean":"2024-03-29T04:33:09.563219+0000","last_became_active":"2024-03-29T04:32:31.593992+0000","last_became_peered":"2024-03-29T04:32:31.593992+0000","last_unstale":"2024-03-29T04:33:09.563219+0000","last_undegraded":"2024-03-29T04:33:09.563219+0000","last_fullsized":"2024-03-29T04:33:09.563219+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:46:08.902905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566419+0000","last_change":"2024-03-29T04:32:31.571035+0000","last_active":"2024-03-29T04:33:09.566419+0000","last_peered":"2024-03-29T04:33:09.566419+0000","last_clean":"2024-03-29T04:33:09.566419+0000","last_became_active":"2024-03-29T04:32:31.570708+0000","last_became_peered":"2024-03-29T04:32:31.570708+0000","last_unstale":"2024-03-29T04:33:09.566419+0000","last_undegraded":"2024-03-29T04:33:09.566419+0000","last_fullsized":"2024-03-29T04:33:09.566419+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:08:54.298826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1300,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1331200,"data_stored":901764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.39700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68400000000000005,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.63200000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.53600000000000003,"15min":0.52600000000000002},"min":{"1min":0.42899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.56999999999999995}]},{"osd":2,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.36899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71799999999999997,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.62},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.55900000000000005,"15min":0.55600000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.53000000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.54300000000000004,"15min":0.53700000000000003},"min":{"1min":0.42899999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55700000000000005,"15min":0.55100000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.70899999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.64200000000000002}]},{"osd":4,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.443,"15min":0.434},"min":{"1min":0.34599999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.442,"15min":0.435},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.46600000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.45700000000000002,"15min":0.45100000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.502,"15min":0.496},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.49099999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50800000000000001,"15min":0.502},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.52000000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462956,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1318912,"data_stored":901676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53500000000000003,"15min":0.52100000000000002},"min":{"1min":0.39600000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72599999999999998,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77100000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.49199999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.55000000000000004,"15min":0.54300000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71899999999999997,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53700000000000003}]},{"osd":2,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.42799999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75900000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58699999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58199999999999996,"15min":0.57699999999999996},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84599999999999997,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.47699999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46600000000000003,"15min":0.45500000000000002},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.46999999999999997,"15min":0.46600000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55800000000000005}]},{"osd":5,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.498,"15min":0.48299999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.504,"15min":0.48799999999999999},"min":{"1min":0.34599999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.57999999999999996}]},{"osd":7,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56100000000000005,"15min":0.56200000000000006},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.746,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.71099999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574916,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":704512,"data_stored":309421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.33700000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.377},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.375,"15min":0.34200000000000003},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.502,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.27500000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.40200000000000002,"15min":0.38400000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.61099999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.33700000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.41799999999999998,"15min":0.436},"min":{"1min":0.27400000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45500000000000002,"15min":0.46700000000000003},"min":{"1min":0.32800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.315}]},{"osd":4,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57899999999999996,"15min":0.57099999999999995},"min":{"1min":0.45200000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80400000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56100000000000005,"15min":0.55600000000000005},"min":{"1min":0.41499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.745,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.48699999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.60599999999999998,"15min":0.60999999999999999},"min":{"1min":0.47599999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.59099999999999997,"15min":0.59799999999999998},"min":{"1min":0.39200000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.498}]},{"osd":6,"last update":"Fri Mar 29 04:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.56599999999999995,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80500000000000005,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.60799999999999998,"15min":0.59599999999999997},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89800000000000002,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.51300000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.62,"15min":0.61699999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.55900000000000005,"15min":0.54700000000000004},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.52700000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738442,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1280,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1310720,"data_stored":900646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.41399999999999998,"15min":0.36499999999999999},"min":{"1min":0.28999999999999998,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.36399999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40400000000000003,"15min":0.40799999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.42599999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.47399999999999998,"15min":0.48699999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.499},{"interface":"front","average":{"1min":0.433,"5min":0.45900000000000002,"15min":0.47399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.47299999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.44900000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.83699999999999997,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.58599999999999997,"15min":0.57599999999999996},"min":{"1min":0.47899999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.90600000000000003,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.57099999999999995}]},{"osd":5,"last update":"Fri Mar 29 04:32:50 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.437,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.85499999999999998,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.623},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64400000000000002,"15min":0.65900000000000003},"min":{"1min":0.42099999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.53600000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.622,"15min":0.628},"min":{"1min":0.44,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76600000000000001,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.60499999999999998,"15min":0.59699999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.86099999999999999,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.55300000000000005}]},{"osd":7,"last update":"Fri Mar 29 04:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.61799999999999999,"15min":0.61199999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.60399999999999998,"15min":0.59799999999999998},"min":{"1min":0.495,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.65500000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411391,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":309684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.377,"15min":0.36799999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52600000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40200000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.45400000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.32300000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.42299999999999999,"15min":0.41799999999999998},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.504,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.47399999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.218}]},{"osd":3,"last update":"Fri Mar 29 04:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.432,"15min":0.46999999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55000000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.495,"15min":0.52500000000000002},"min":{"1min":0.35999999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.64800000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.312}]},{"osd":4,"last update":"Fri Mar 29 04:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59799999999999998,"15min":0.61199999999999999},"min":{"1min":0.433,"5min":0.376,"15min":0.376},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65000000000000002,"15min":0.66100000000000003},"min":{"1min":0.495,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.63100000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:32:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.625},"min":{"1min":0.45400000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70099999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56899999999999995,"15min":0.57899999999999996},"min":{"1min":0.432,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.66200000000000003,"5min":0.75,"15min":0.75},"last":0.66300000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:33:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44800000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.85299999999999998,"5min":1.01,"15min":1.01},"last":0.69699999999999995}]},{"osd":7,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.495,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.71699999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149754,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.38400000000000001,"15min":0.39400000000000002},"min":{"1min":0.249,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.63200000000000001,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.39900000000000002,"15min":0.40200000000000002},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55800000000000005,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.32000000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.41899999999999998,"15min":0.42899999999999999},"min":{"1min":0.26600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.61399999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.373,"5min":0.41099999999999998,"15min":0.42099999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.58199999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.377}]},{"osd":2,"last update":"Fri Mar 29 04:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.435,"15min":0.44500000000000001},"min":{"1min":0.29699999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.57699999999999996,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.42999999999999999,"15min":0.434},"min":{"1min":0.29099999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56399999999999995,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.36499999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70199999999999996,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.44600000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72099999999999997,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.66300000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:32:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.439,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.64600000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.878,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59599999999999997,"15min":0.59399999999999997},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83199999999999996,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.623}]},{"osd":7,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41199999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81100000000000005,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.70599999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986229,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":311384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.52700000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72199999999999998,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.52500000000000002,"15min":0.501},"min":{"1min":0.39400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69899999999999995,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.69899999999999995}]},{"osd":1,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52500000000000002,"15min":0.51200000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68899999999999995,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53400000000000003,"15min":0.51800000000000002},"min":{"1min":0.35699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73099999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55900000000000005}]},{"osd":2,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.52300000000000002},"min":{"1min":0.34699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65700000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.58399999999999996}]},{"osd":3,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.54700000000000004,"15min":0.53700000000000003},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71599999999999997,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55100000000000005,"15min":0.53300000000000003},"min":{"1min":0.434,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76200000000000001,"5min":0.874,"15min":0.874},"last":0.44400000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.438,"15min":0.41399999999999998},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69499999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.47899999999999998,"15min":0.44700000000000001},"min":{"1min":0.374,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.65500000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.44500000000000001,"15min":0.41899999999999998},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65600000000000003,"5min":0.745,"15min":0.745},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.45300000000000001,"15min":0.43099999999999999},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.66900000000000004}]},{"osd":7,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.52400000000000002,"15min":0.51800000000000002},"min":{"1min":0.38500000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.621},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.52700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34399999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.749}]}]},{"osd":5,"up_from":39,"seq":167503724592,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.504,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.628,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.57699999999999996}]},{"osd":1,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.67400000000000004,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.41199999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.75600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.622}]},{"osd":2,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.56799999999999995,"15min":0.57499999999999996},"min":{"1min":0.38700000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81200000000000006,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.60999999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39700000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.64600000000000002,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.378,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78400000000000003,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.64700000000000002}]},{"osd":4,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.434,"15min":0.41599999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60999999999999999,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.44900000000000001,"15min":0.41999999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.64600000000000002,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.59299999999999997}]},{"osd":6,"last update":"Fri Mar 29 04:33:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.46899999999999997,"15min":0.44500000000000001},"min":{"1min":0.317,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71299999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.50700000000000001,"15min":0.47799999999999998},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69699999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.504}]},{"osd":7,"last update":"Fri Mar 29 04:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.67800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:33:34.491 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:33:34.598 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:33:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:33:34.776 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:34 smithi029 ceph-mon[30044]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:34.776 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:34 smithi029 ceph-mon[41899]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:34.853 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:34 smithi099 ceph-mon[33865]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:35.097 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:35.902 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:35 smithi099 ceph-mon[33865]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:33:35.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:35 smithi029 ceph-mon[30044]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:33:35.923 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:35 smithi029 ceph-mon[41899]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:33:36.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:36 smithi029 ceph-mon[30044]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:36.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:36.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:36 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:36.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:36 smithi029 ceph-mon[41899]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:36.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:36.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:36 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:37.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:36 smithi099 ceph-mon[33865]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:37.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:37.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:36 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:37.599 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:33:37.600 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-03-29T04:33:38.466 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-03-29T04:33:37.451701+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386236,"kb_used_data":7368,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574980,"statfs":{"total":767960285184,"available":765516779520,"internally_reserved":0,"allocated":7544832,"data_stored":4254884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002556"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566330+0000","last_change":"2024-03-29T04:32:31.592888+0000","last_active":"2024-03-29T04:33:09.566330+0000","last_peered":"2024-03-29T04:33:09.566330+0000","last_clean":"2024-03-29T04:33:09.566330+0000","last_became_active":"2024-03-29T04:32:31.592647+0000","last_became_peered":"2024-03-29T04:32:31.592647+0000","last_unstale":"2024-03-29T04:33:09.566330+0000","last_undegraded":"2024-03-29T04:33:09.566330+0000","last_fullsized":"2024-03-29T04:33:09.566330+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:49:59.071894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056784+0000","last_change":"2024-03-29T04:32:31.576106+0000","last_active":"2024-03-29T04:33:10.056784+0000","last_peered":"2024-03-29T04:33:10.056784+0000","last_clean":"2024-03-29T04:33:10.056784+0000","last_became_active":"2024-03-29T04:32:31.575248+0000","last_became_peered":"2024-03-29T04:32:31.575248+0000","last_unstale":"2024-03-29T04:33:10.056784+0000","last_undegraded":"2024-03-29T04:33:10.056784+0000","last_fullsized":"2024-03-29T04:33:10.056784+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:48:46.726681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560808+0000","last_change":"2024-03-29T04:32:31.590152+0000","last_active":"2024-03-29T04:33:09.560808+0000","last_peered":"2024-03-29T04:33:09.560808+0000","last_clean":"2024-03-29T04:33:09.560808+0000","last_became_active":"2024-03-29T04:32:31.589845+0000","last_became_peered":"2024-03-29T04:32:31.589845+0000","last_unstale":"2024-03-29T04:33:09.560808+0000","last_undegraded":"2024-03-29T04:33:09.560808+0000","last_fullsized":"2024-03-29T04:33:09.560808+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:38:38.889964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.562164+0000","last_change":"2024-03-29T04:32:31.565434+0000","last_active":"2024-03-29T04:33:09.562164+0000","last_peered":"2024-03-29T04:33:09.562164+0000","last_clean":"2024-03-29T04:33:09.562164+0000","last_became_active":"2024-03-29T04:32:31.562325+0000","last_became_peered":"2024-03-29T04:32:31.562325+0000","last_unstale":"2024-03-29T04:33:09.562164+0000","last_undegraded":"2024-03-29T04:33:09.562164+0000","last_fullsized":"2024-03-29T04:33:09.562164+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:43:20.651381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566580+0000","last_change":"2024-03-29T04:32:31.589669+0000","last_active":"2024-03-29T04:33:09.566580+0000","last_peered":"2024-03-29T04:33:09.566580+0000","last_clean":"2024-03-29T04:33:09.566580+0000","last_became_active":"2024-03-29T04:32:31.589437+0000","last_became_peered":"2024-03-29T04:32:31.589437+0000","last_unstale":"2024-03-29T04:33:09.566580+0000","last_undegraded":"2024-03-29T04:33:09.566580+0000","last_fullsized":"2024-03-29T04:33:09.566580+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:26:27.350025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560922+0000","last_change":"2024-03-29T04:32:31.572333+0000","last_active":"2024-03-29T04:33:09.560922+0000","last_peered":"2024-03-29T04:33:09.560922+0000","last_clean":"2024-03-29T04:33:09.560922+0000","last_became_active":"2024-03-29T04:32:31.572050+0000","last_became_peered":"2024-03-29T04:32:31.572050+0000","last_unstale":"2024-03-29T04:33:09.560922+0000","last_undegraded":"2024-03-29T04:33:09.560922+0000","last_fullsized":"2024-03-29T04:33:09.560922+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:41:46.118602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566716+0000","last_change":"2024-03-29T04:32:31.599296+0000","last_active":"2024-03-29T04:33:09.566716+0000","last_peered":"2024-03-29T04:33:09.566716+0000","last_clean":"2024-03-29T04:33:09.566716+0000","last_became_active":"2024-03-29T04:32:31.598999+0000","last_became_peered":"2024-03-29T04:32:31.598999+0000","last_unstale":"2024-03-29T04:33:09.566716+0000","last_undegraded":"2024-03-29T04:33:09.566716+0000","last_fullsized":"2024-03-29T04:33:09.566716+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:52:11.559051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560733+0000","last_change":"2024-03-29T04:32:31.572619+0000","last_active":"2024-03-29T04:33:09.560733+0000","last_peered":"2024-03-29T04:33:09.560733+0000","last_clean":"2024-03-29T04:33:09.560733+0000","last_became_active":"2024-03-29T04:32:31.572303+0000","last_became_peered":"2024-03-29T04:32:31.572303+0000","last_unstale":"2024-03-29T04:33:09.560733+0000","last_undegraded":"2024-03-29T04:33:09.560733+0000","last_fullsized":"2024-03-29T04:33:09.560733+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:55:51.124870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561128+0000","last_change":"2024-03-29T04:32:31.595348+0000","last_active":"2024-03-29T04:33:09.561128+0000","last_peered":"2024-03-29T04:33:09.561128+0000","last_clean":"2024-03-29T04:33:09.561128+0000","last_became_active":"2024-03-29T04:32:31.595059+0000","last_became_peered":"2024-03-29T04:32:31.595059+0000","last_unstale":"2024-03-29T04:33:09.561128+0000","last_undegraded":"2024-03-29T04:33:09.561128+0000","last_fullsized":"2024-03-29T04:33:09.561128+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:28:08.453900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056649+0000","last_change":"2024-03-29T04:32:31.581209+0000","last_active":"2024-03-29T04:33:10.056649+0000","last_peered":"2024-03-29T04:33:10.056649+0000","last_clean":"2024-03-29T04:33:10.056649+0000","last_became_active":"2024-03-29T04:32:31.580935+0000","last_became_peered":"2024-03-29T04:32:31.580935+0000","last_unstale":"2024-03-29T04:33:10.056649+0000","last_undegraded":"2024-03-29T04:33:10.056649+0000","last_fullsized":"2024-03-29T04:33:10.056649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:18:36.520402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566878+0000","last_change":"2024-03-29T04:32:31.588347+0000","last_active":"2024-03-29T04:33:09.566878+0000","last_peered":"2024-03-29T04:33:09.566878+0000","last_clean":"2024-03-29T04:33:09.566878+0000","last_became_active":"2024-03-29T04:32:31.588080+0000","last_became_peered":"2024-03-29T04:32:31.588080+0000","last_unstale":"2024-03-29T04:33:09.566878+0000","last_undegraded":"2024-03-29T04:33:09.566878+0000","last_fullsized":"2024-03-29T04:33:09.566878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:52:40.449275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056935+0000","last_change":"2024-03-29T04:32:31.578558+0000","last_active":"2024-03-29T04:33:10.056935+0000","last_peered":"2024-03-29T04:33:10.056935+0000","last_clean":"2024-03-29T04:33:10.056935+0000","last_became_active":"2024-03-29T04:32:31.578252+0000","last_became_peered":"2024-03-29T04:32:31.578252+0000","last_unstale":"2024-03-29T04:33:10.056935+0000","last_undegraded":"2024-03-29T04:33:10.056935+0000","last_fullsized":"2024-03-29T04:33:10.056935+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:27:05.696647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568774+0000","last_change":"2024-03-29T04:31:14.925966+0000","last_active":"2024-03-29T04:33:09.568774+0000","last_peered":"2024-03-29T04:33:09.568774+0000","last_clean":"2024-03-29T04:33:09.568774+0000","last_became_active":"2024-03-29T04:31:14.925528+0000","last_became_peered":"2024-03-29T04:31:14.925528+0000","last_unstale":"2024-03-29T04:33:09.568774+0000","last_undegraded":"2024-03-29T04:33:09.568774+0000","last_fullsized":"2024-03-29T04:33:09.568774+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:30:41.616775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560275+0000","last_change":"2024-03-29T04:32:31.772928+0000","last_active":"2024-03-29T04:33:09.560275+0000","last_peered":"2024-03-29T04:33:09.560275+0000","last_clean":"2024-03-29T04:33:09.560275+0000","last_became_active":"2024-03-29T04:32:31.772379+0000","last_became_peered":"2024-03-29T04:32:31.772379+0000","last_unstale":"2024-03-29T04:33:09.560275+0000","last_undegraded":"2024-03-29T04:33:09.560275+0000","last_fullsized":"2024-03-29T04:33:09.560275+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:43:56.698808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567540+0000","last_change":"2024-03-29T04:31:15.188305+0000","last_active":"2024-03-29T04:33:09.567540+0000","last_peered":"2024-03-29T04:33:09.567540+0000","last_clean":"2024-03-29T04:33:09.567540+0000","last_became_active":"2024-03-29T04:31:15.187906+0000","last_became_peered":"2024-03-29T04:31:15.187906+0000","last_unstale":"2024-03-29T04:33:09.567540+0000","last_undegraded":"2024-03-29T04:33:09.567540+0000","last_fullsized":"2024-03-29T04:33:09.567540+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:03:13.618806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.562932+0000","last_change":"2024-03-29T04:32:31.573515+0000","last_active":"2024-03-29T04:33:09.562932+0000","last_peered":"2024-03-29T04:33:09.562932+0000","last_clean":"2024-03-29T04:33:09.562932+0000","last_became_active":"2024-03-29T04:32:31.573229+0000","last_became_peered":"2024-03-29T04:32:31.573229+0000","last_unstale":"2024-03-29T04:33:09.562932+0000","last_undegraded":"2024-03-29T04:33:09.562932+0000","last_fullsized":"2024-03-29T04:33:09.562932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:49:46.508547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056524+0000","last_change":"2024-03-29T04:32:31.772599+0000","last_active":"2024-03-29T04:33:10.056524+0000","last_peered":"2024-03-29T04:33:10.056524+0000","last_clean":"2024-03-29T04:33:10.056524+0000","last_became_active":"2024-03-29T04:32:31.772236+0000","last_became_peered":"2024-03-29T04:32:31.772236+0000","last_unstale":"2024-03-29T04:33:10.056524+0000","last_undegraded":"2024-03-29T04:33:10.056524+0000","last_fullsized":"2024-03-29T04:33:10.056524+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:15:53.964412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568934+0000","last_change":"2024-03-29T04:32:31.573147+0000","last_active":"2024-03-29T04:33:09.568934+0000","last_peered":"2024-03-29T04:33:09.568934+0000","last_clean":"2024-03-29T04:33:09.568934+0000","last_became_active":"2024-03-29T04:32:31.572509+0000","last_became_peered":"2024-03-29T04:32:31.572509+0000","last_unstale":"2024-03-29T04:33:09.568934+0000","last_undegraded":"2024-03-29T04:33:09.568934+0000","last_fullsized":"2024-03-29T04:33:09.568934+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:40:56.358964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565966+0000","last_change":"2024-03-29T04:31:15.188174+0000","last_active":"2024-03-29T04:33:09.565966+0000","last_peered":"2024-03-29T04:33:09.565966+0000","last_clean":"2024-03-29T04:33:09.565966+0000","last_became_active":"2024-03-29T04:31:15.187629+0000","last_became_peered":"2024-03-29T04:31:15.187629+0000","last_unstale":"2024-03-29T04:33:09.565966+0000","last_undegraded":"2024-03-29T04:33:09.565966+0000","last_fullsized":"2024-03-29T04:33:09.565966+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:47:42.088021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.057001+0000","last_change":"2024-03-29T04:32:31.595358+0000","last_active":"2024-03-29T04:33:10.057001+0000","last_peered":"2024-03-29T04:33:10.057001+0000","last_clean":"2024-03-29T04:33:10.057001+0000","last_became_active":"2024-03-29T04:32:31.595051+0000","last_became_peered":"2024-03-29T04:32:31.595051+0000","last_unstale":"2024-03-29T04:33:10.057001+0000","last_undegraded":"2024-03-29T04:33:10.057001+0000","last_fullsized":"2024-03-29T04:33:10.057001+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:15:09.145501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056407+0000","last_change":"2024-03-29T04:32:31.574916+0000","last_active":"2024-03-29T04:33:10.056407+0000","last_peered":"2024-03-29T04:33:10.056407+0000","last_clean":"2024-03-29T04:33:10.056407+0000","last_became_active":"2024-03-29T04:32:31.574627+0000","last_became_peered":"2024-03-29T04:32:31.574627+0000","last_unstale":"2024-03-29T04:33:10.056407+0000","last_undegraded":"2024-03-29T04:33:10.056407+0000","last_fullsized":"2024-03-29T04:33:10.056407+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:45:31.424895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565826+0000","last_change":"2024-03-29T04:31:14.926407+0000","last_active":"2024-03-29T04:33:09.565826+0000","last_peered":"2024-03-29T04:33:09.565826+0000","last_clean":"2024-03-29T04:33:09.565826+0000","last_became_active":"2024-03-29T04:31:14.926060+0000","last_became_peered":"2024-03-29T04:31:14.926060+0000","last_unstale":"2024-03-29T04:33:09.565826+0000","last_undegraded":"2024-03-29T04:33:09.565826+0000","last_fullsized":"2024-03-29T04:33:09.565826+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:30.113952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568623+0000","last_change":"2024-03-29T04:32:31.772849+0000","last_active":"2024-03-29T04:33:09.568623+0000","last_peered":"2024-03-29T04:33:09.568623+0000","last_clean":"2024-03-29T04:33:09.568623+0000","last_became_active":"2024-03-29T04:32:31.772561+0000","last_became_peered":"2024-03-29T04:32:31.772561+0000","last_unstale":"2024-03-29T04:33:09.568623+0000","last_undegraded":"2024-03-29T04:33:09.568623+0000","last_fullsized":"2024-03-29T04:33:09.568623+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:31.653522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561123+0000","last_change":"2024-03-29T04:32:31.567152+0000","last_active":"2024-03-29T04:33:09.561123+0000","last_peered":"2024-03-29T04:33:09.561123+0000","last_clean":"2024-03-29T04:33:09.561123+0000","last_became_active":"2024-03-29T04:32:31.566871+0000","last_became_peered":"2024-03-29T04:32:31.566871+0000","last_unstale":"2024-03-29T04:33:09.561123+0000","last_undegraded":"2024-03-29T04:33:09.561123+0000","last_fullsized":"2024-03-29T04:33:09.561123+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:44:25.854105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565705+0000","last_change":"2024-03-29T04:30:40.532165+0000","last_active":"2024-03-29T04:33:09.565705+0000","last_peered":"2024-03-29T04:33:09.565705+0000","last_clean":"2024-03-29T04:33:09.565705+0000","last_became_active":"2024-03-29T04:30:40.531726+0000","last_became_peered":"2024-03-29T04:30:40.531726+0000","last_unstale":"2024-03-29T04:33:09.565705+0000","last_undegraded":"2024-03-29T04:33:09.565705+0000","last_fullsized":"2024-03-29T04:33:09.565705+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_clean_scrub_stamp":"2024-03-29T04:28:33.017662+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:06:56.360284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560684+0000","last_change":"2024-03-29T04:31:14.925369+0000","last_active":"2024-03-29T04:33:09.560684+0000","last_peered":"2024-03-29T04:33:09.560684+0000","last_clean":"2024-03-29T04:33:09.560684+0000","last_became_active":"2024-03-29T04:31:14.924765+0000","last_became_peered":"2024-03-29T04:31:14.924765+0000","last_unstale":"2024-03-29T04:33:09.560684+0000","last_undegraded":"2024-03-29T04:33:09.560684+0000","last_fullsized":"2024-03-29T04:33:09.560684+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:59:12.817095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567674+0000","last_change":"2024-03-29T04:32:31.597573+0000","last_active":"2024-03-29T04:33:09.567674+0000","last_peered":"2024-03-29T04:33:09.567674+0000","last_clean":"2024-03-29T04:33:09.567674+0000","last_became_active":"2024-03-29T04:32:31.596766+0000","last_became_peered":"2024-03-29T04:32:31.596766+0000","last_unstale":"2024-03-29T04:33:09.567674+0000","last_undegraded":"2024-03-29T04:33:09.567674+0000","last_fullsized":"2024-03-29T04:33:09.567674+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:47:45.530721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567140+0000","last_change":"2024-03-29T04:32:31.567031+0000","last_active":"2024-03-29T04:33:09.567140+0000","last_peered":"2024-03-29T04:33:09.567140+0000","last_clean":"2024-03-29T04:33:09.567140+0000","last_became_active":"2024-03-29T04:32:31.566719+0000","last_became_peered":"2024-03-29T04:32:31.566719+0000","last_unstale":"2024-03-29T04:33:09.567140+0000","last_undegraded":"2024-03-29T04:33:09.567140+0000","last_fullsized":"2024-03-29T04:33:09.567140+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:17:38.096048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561995+0000","last_change":"2024-03-29T04:31:14.925939+0000","last_active":"2024-03-29T04:33:09.561995+0000","last_peered":"2024-03-29T04:33:09.561995+0000","last_clean":"2024-03-29T04:33:09.561995+0000","last_became_active":"2024-03-29T04:31:14.925609+0000","last_became_peered":"2024-03-29T04:31:14.925609+0000","last_unstale":"2024-03-29T04:33:09.561995+0000","last_undegraded":"2024-03-29T04:33:09.561995+0000","last_fullsized":"2024-03-29T04:33:09.561995+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:51.860391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568405+0000","last_change":"2024-03-29T04:32:31.598332+0000","last_active":"2024-03-29T04:33:09.568405+0000","last_peered":"2024-03-29T04:33:09.568405+0000","last_clean":"2024-03-29T04:33:09.568405+0000","last_became_active":"2024-03-29T04:32:31.597204+0000","last_became_peered":"2024-03-29T04:32:31.597204+0000","last_unstale":"2024-03-29T04:33:09.568405+0000","last_undegraded":"2024-03-29T04:33:09.568405+0000","last_fullsized":"2024-03-29T04:33:09.568405+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:46:05.313560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566621+0000","last_change":"2024-03-29T04:32:31.574292+0000","last_active":"2024-03-29T04:33:09.566621+0000","last_peered":"2024-03-29T04:33:09.566621+0000","last_clean":"2024-03-29T04:33:09.566621+0000","last_became_active":"2024-03-29T04:32:31.573679+0000","last_became_peered":"2024-03-29T04:32:31.573679+0000","last_unstale":"2024-03-29T04:33:09.566621+0000","last_undegraded":"2024-03-29T04:33:09.566621+0000","last_fullsized":"2024-03-29T04:33:09.566621+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:51:01.779754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560612+0000","last_change":"2024-03-29T04:31:14.924983+0000","last_active":"2024-03-29T04:33:09.560612+0000","last_peered":"2024-03-29T04:33:09.560612+0000","last_clean":"2024-03-29T04:33:09.560612+0000","last_became_active":"2024-03-29T04:31:14.924603+0000","last_became_peered":"2024-03-29T04:31:14.924603+0000","last_unstale":"2024-03-29T04:33:09.560612+0000","last_undegraded":"2024-03-29T04:33:09.560612+0000","last_fullsized":"2024-03-29T04:33:09.560612+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:39:19.333479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567398+0000","last_change":"2024-03-29T04:32:31.599616+0000","last_active":"2024-03-29T04:33:09.567398+0000","last_peered":"2024-03-29T04:33:09.567398+0000","last_clean":"2024-03-29T04:33:09.567398+0000","last_became_active":"2024-03-29T04:32:31.599131+0000","last_became_peered":"2024-03-29T04:32:31.599131+0000","last_unstale":"2024-03-29T04:33:09.567398+0000","last_undegraded":"2024-03-29T04:33:09.567398+0000","last_fullsized":"2024-03-29T04:33:09.567398+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:09:17.407227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560907+0000","last_change":"2024-03-29T04:32:31.576596+0000","last_active":"2024-03-29T04:33:09.560907+0000","last_peered":"2024-03-29T04:33:09.560907+0000","last_clean":"2024-03-29T04:33:09.560907+0000","last_became_active":"2024-03-29T04:32:31.576298+0000","last_became_peered":"2024-03-29T04:32:31.576298+0000","last_unstale":"2024-03-29T04:33:09.560907+0000","last_undegraded":"2024-03-29T04:33:09.560907+0000","last_fullsized":"2024-03-29T04:33:09.560907+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:19:56.607906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568324+0000","last_change":"2024-03-29T04:31:14.925317+0000","last_active":"2024-03-29T04:33:09.568324+0000","last_peered":"2024-03-29T04:33:09.568324+0000","last_clean":"2024-03-29T04:33:09.568324+0000","last_became_active":"2024-03-29T04:31:14.925065+0000","last_became_peered":"2024-03-29T04:31:14.925065+0000","last_unstale":"2024-03-29T04:33:09.568324+0000","last_undegraded":"2024-03-29T04:33:09.568324+0000","last_fullsized":"2024-03-29T04:33:09.568324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:11.187076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:33:11.002270+0000","last_change":"2024-03-29T04:33:11.002270+0000","last_active":"2024-03-29T04:33:11.002270+0000","last_peered":"2024-03-29T04:33:11.002270+0000","last_clean":"2024-03-29T04:33:11.002270+0000","last_became_active":"2024-03-29T04:33:11.001967+0000","last_became_peered":"2024-03-29T04:33:11.001967+0000","last_unstale":"2024-03-29T04:33:11.002270+0000","last_undegraded":"2024-03-29T04:33:11.002270+0000","last_fullsized":"2024-03-29T04:33:11.002270+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:45:46.932749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560414+0000","last_change":"2024-03-29T04:32:31.571365+0000","last_active":"2024-03-29T04:33:09.560414+0000","last_peered":"2024-03-29T04:33:09.560414+0000","last_clean":"2024-03-29T04:33:09.560414+0000","last_became_active":"2024-03-29T04:32:31.570932+0000","last_became_peered":"2024-03-29T04:32:31.570932+0000","last_unstale":"2024-03-29T04:33:09.560414+0000","last_undegraded":"2024-03-29T04:33:09.560414+0000","last_fullsized":"2024-03-29T04:33:09.560414+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:17:21.103008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567154+0000","last_change":"2024-03-29T04:32:31.593605+0000","last_active":"2024-03-29T04:33:09.567154+0000","last_peered":"2024-03-29T04:33:09.567154+0000","last_clean":"2024-03-29T04:33:09.567154+0000","last_became_active":"2024-03-29T04:32:31.593330+0000","last_became_peered":"2024-03-29T04:32:31.593330+0000","last_unstale":"2024-03-29T04:33:09.567154+0000","last_undegraded":"2024-03-29T04:33:09.567154+0000","last_fullsized":"2024-03-29T04:33:09.567154+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:40.015461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566969+0000","last_change":"2024-03-29T04:32:31.568721+0000","last_active":"2024-03-29T04:33:09.566969+0000","last_peered":"2024-03-29T04:33:09.566969+0000","last_clean":"2024-03-29T04:33:09.566969+0000","last_became_active":"2024-03-29T04:32:31.568416+0000","last_became_peered":"2024-03-29T04:32:31.568416+0000","last_unstale":"2024-03-29T04:33:09.566969+0000","last_undegraded":"2024-03-29T04:33:09.566969+0000","last_fullsized":"2024-03-29T04:33:09.566969+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:15:38.580168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567250+0000","last_change":"2024-03-29T04:32:31.593457+0000","last_active":"2024-03-29T04:33:09.567250+0000","last_peered":"2024-03-29T04:33:09.567250+0000","last_clean":"2024-03-29T04:33:09.567250+0000","last_became_active":"2024-03-29T04:32:31.593199+0000","last_became_peered":"2024-03-29T04:32:31.593199+0000","last_unstale":"2024-03-29T04:33:09.567250+0000","last_undegraded":"2024-03-29T04:33:09.567250+0000","last_fullsized":"2024-03-29T04:33:09.567250+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:08:56.455967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568562+0000","last_change":"2024-03-29T04:32:31.574602+0000","last_active":"2024-03-29T04:33:09.568562+0000","last_peered":"2024-03-29T04:33:09.568562+0000","last_clean":"2024-03-29T04:33:09.568562+0000","last_became_active":"2024-03-29T04:32:31.574341+0000","last_became_peered":"2024-03-29T04:32:31.574341+0000","last_unstale":"2024-03-29T04:33:09.568562+0000","last_undegraded":"2024-03-29T04:33:09.568562+0000","last_fullsized":"2024-03-29T04:33:09.568562+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:41:48.123074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567005+0000","last_change":"2024-03-29T04:32:31.589205+0000","last_active":"2024-03-29T04:33:09.567005+0000","last_peered":"2024-03-29T04:33:09.567005+0000","last_clean":"2024-03-29T04:33:09.567005+0000","last_became_active":"2024-03-29T04:32:31.588852+0000","last_became_peered":"2024-03-29T04:32:31.588852+0000","last_unstale":"2024-03-29T04:33:09.567005+0000","last_undegraded":"2024-03-29T04:33:09.567005+0000","last_fullsized":"2024-03-29T04:33:09.567005+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:02:51.103250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561274+0000","last_change":"2024-03-29T04:32:31.573299+0000","last_active":"2024-03-29T04:33:09.561274+0000","last_peered":"2024-03-29T04:33:09.561274+0000","last_clean":"2024-03-29T04:33:09.561274+0000","last_became_active":"2024-03-29T04:32:31.572968+0000","last_became_peered":"2024-03-29T04:32:31.572968+0000","last_unstale":"2024-03-29T04:33:09.561274+0000","last_undegraded":"2024-03-29T04:33:09.561274+0000","last_fullsized":"2024-03-29T04:33:09.561274+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:31:45.450843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056665+0000","last_change":"2024-03-29T04:32:31.595919+0000","last_active":"2024-03-29T04:33:10.056665+0000","last_peered":"2024-03-29T04:33:10.056665+0000","last_clean":"2024-03-29T04:33:10.056665+0000","last_became_active":"2024-03-29T04:32:31.595623+0000","last_became_peered":"2024-03-29T04:32:31.595623+0000","last_unstale":"2024-03-29T04:33:10.056665+0000","last_undegraded":"2024-03-29T04:33:10.056665+0000","last_fullsized":"2024-03-29T04:33:10.056665+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:26:11.492507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560820+0000","last_change":"2024-03-29T04:32:31.575921+0000","last_active":"2024-03-29T04:33:09.560820+0000","last_peered":"2024-03-29T04:33:09.560820+0000","last_clean":"2024-03-29T04:33:09.560820+0000","last_became_active":"2024-03-29T04:32:31.575668+0000","last_became_peered":"2024-03-29T04:32:31.575668+0000","last_unstale":"2024-03-29T04:33:09.560820+0000","last_undegraded":"2024-03-29T04:33:09.560820+0000","last_fullsized":"2024-03-29T04:33:09.560820+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:07:35.477597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.720781+0000","last_change":"2024-03-29T04:33:10.720781+0000","last_active":"2024-03-29T04:33:10.720781+0000","last_peered":"2024-03-29T04:33:10.720781+0000","last_clean":"2024-03-29T04:33:10.720781+0000","last_became_active":"2024-03-29T04:33:10.720378+0000","last_became_peered":"2024-03-29T04:33:10.720378+0000","last_unstale":"2024-03-29T04:33:10.720781+0000","last_undegraded":"2024-03-29T04:33:10.720781+0000","last_fullsized":"2024-03-29T04:33:10.720781+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:20:45.090317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566125+0000","last_change":"2024-03-29T04:32:31.573699+0000","last_active":"2024-03-29T04:33:09.566125+0000","last_peered":"2024-03-29T04:33:09.566125+0000","last_clean":"2024-03-29T04:33:09.566125+0000","last_became_active":"2024-03-29T04:32:31.573265+0000","last_became_peered":"2024-03-29T04:32:31.573265+0000","last_unstale":"2024-03-29T04:33:09.566125+0000","last_undegraded":"2024-03-29T04:33:09.566125+0000","last_fullsized":"2024-03-29T04:33:09.566125+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:05:16.290933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566678+0000","last_change":"2024-03-29T04:32:31.593225+0000","last_active":"2024-03-29T04:33:09.566678+0000","last_peered":"2024-03-29T04:33:09.566678+0000","last_clean":"2024-03-29T04:33:09.566678+0000","last_became_active":"2024-03-29T04:32:31.592627+0000","last_became_peered":"2024-03-29T04:32:31.592627+0000","last_unstale":"2024-03-29T04:33:09.566678+0000","last_undegraded":"2024-03-29T04:33:09.566678+0000","last_fullsized":"2024-03-29T04:33:09.566678+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:47:14.028885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566024+0000","last_change":"2024-03-29T04:32:31.586692+0000","last_active":"2024-03-29T04:33:09.566024+0000","last_peered":"2024-03-29T04:33:09.566024+0000","last_clean":"2024-03-29T04:33:09.566024+0000","last_became_active":"2024-03-29T04:32:31.586390+0000","last_became_peered":"2024-03-29T04:32:31.586390+0000","last_unstale":"2024-03-29T04:33:09.566024+0000","last_undegraded":"2024-03-29T04:33:09.566024+0000","last_fullsized":"2024-03-29T04:33:09.566024+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:14:19.075337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560569+0000","last_change":"2024-03-29T04:32:31.772869+0000","last_active":"2024-03-29T04:33:09.560569+0000","last_peered":"2024-03-29T04:33:09.560569+0000","last_clean":"2024-03-29T04:33:09.560569+0000","last_became_active":"2024-03-29T04:32:31.772518+0000","last_became_peered":"2024-03-29T04:32:31.772518+0000","last_unstale":"2024-03-29T04:33:09.560569+0000","last_undegraded":"2024-03-29T04:33:09.560569+0000","last_fullsized":"2024-03-29T04:33:09.560569+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:03:29.625832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566052+0000","last_change":"2024-03-29T04:32:31.580267+0000","last_active":"2024-03-29T04:33:09.566052+0000","last_peered":"2024-03-29T04:33:09.566052+0000","last_clean":"2024-03-29T04:33:09.566052+0000","last_became_active":"2024-03-29T04:32:31.580016+0000","last_became_peered":"2024-03-29T04:32:31.580016+0000","last_unstale":"2024-03-29T04:33:09.566052+0000","last_undegraded":"2024-03-29T04:33:09.566052+0000","last_fullsized":"2024-03-29T04:33:09.566052+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:35:44.602037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.563012+0000","last_change":"2024-03-29T04:32:31.772688+0000","last_active":"2024-03-29T04:33:09.563012+0000","last_peered":"2024-03-29T04:33:09.563012+0000","last_clean":"2024-03-29T04:33:09.563012+0000","last_became_active":"2024-03-29T04:32:31.772378+0000","last_became_peered":"2024-03-29T04:32:31.772378+0000","last_unstale":"2024-03-29T04:33:09.563012+0000","last_undegraded":"2024-03-29T04:33:09.563012+0000","last_fullsized":"2024-03-29T04:33:09.563012+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:41:34.928801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568471+0000","last_change":"2024-03-29T04:32:31.570678+0000","last_active":"2024-03-29T04:33:09.568471+0000","last_peered":"2024-03-29T04:33:09.568471+0000","last_clean":"2024-03-29T04:33:09.568471+0000","last_became_active":"2024-03-29T04:32:31.570015+0000","last_became_peered":"2024-03-29T04:32:31.570015+0000","last_unstale":"2024-03-29T04:33:09.568471+0000","last_undegraded":"2024-03-29T04:33:09.568471+0000","last_fullsized":"2024-03-29T04:33:09.568471+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:19:55.110568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056849+0000","last_change":"2024-03-29T04:32:31.773532+0000","last_active":"2024-03-29T04:33:10.056849+0000","last_peered":"2024-03-29T04:33:10.056849+0000","last_clean":"2024-03-29T04:33:10.056849+0000","last_became_active":"2024-03-29T04:32:31.773256+0000","last_became_peered":"2024-03-29T04:32:31.773256+0000","last_unstale":"2024-03-29T04:33:10.056849+0000","last_undegraded":"2024-03-29T04:33:10.056849+0000","last_fullsized":"2024-03-29T04:33:10.056849+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:46:29.922937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.565650+0000","last_change":"2024-03-29T04:32:31.578200+0000","last_active":"2024-03-29T04:33:09.565650+0000","last_peered":"2024-03-29T04:33:09.565650+0000","last_clean":"2024-03-29T04:33:09.565650+0000","last_became_active":"2024-03-29T04:32:31.577900+0000","last_became_peered":"2024-03-29T04:32:31.577900+0000","last_unstale":"2024-03-29T04:33:09.565650+0000","last_undegraded":"2024-03-29T04:33:09.565650+0000","last_fullsized":"2024-03-29T04:33:09.565650+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:04:23.132474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560788+0000","last_change":"2024-03-29T04:32:31.774941+0000","last_active":"2024-03-29T04:33:09.560788+0000","last_peered":"2024-03-29T04:33:09.560788+0000","last_clean":"2024-03-29T04:33:09.560788+0000","last_became_active":"2024-03-29T04:32:31.774665+0000","last_became_peered":"2024-03-29T04:32:31.774665+0000","last_unstale":"2024-03-29T04:33:09.560788+0000","last_undegraded":"2024-03-29T04:33:09.560788+0000","last_fullsized":"2024-03-29T04:33:09.560788+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:16:08.260190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056529+0000","last_change":"2024-03-29T04:32:31.577802+0000","last_active":"2024-03-29T04:33:10.056529+0000","last_peered":"2024-03-29T04:33:10.056529+0000","last_clean":"2024-03-29T04:33:10.056529+0000","last_became_active":"2024-03-29T04:32:31.577475+0000","last_became_peered":"2024-03-29T04:32:31.577475+0000","last_unstale":"2024-03-29T04:33:10.056529+0000","last_undegraded":"2024-03-29T04:33:10.056529+0000","last_fullsized":"2024-03-29T04:33:10.056529+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:25:09.560054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.720797+0000","last_change":"2024-03-29T04:33:10.720797+0000","last_active":"2024-03-29T04:33:10.720797+0000","last_peered":"2024-03-29T04:33:10.720797+0000","last_clean":"2024-03-29T04:33:10.720797+0000","last_became_active":"2024-03-29T04:33:10.720516+0000","last_became_peered":"2024-03-29T04:33:10.720516+0000","last_unstale":"2024-03-29T04:33:10.720797+0000","last_undegraded":"2024-03-29T04:33:10.720797+0000","last_fullsized":"2024-03-29T04:33:10.720797+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:37:11.811412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566175+0000","last_change":"2024-03-29T04:32:31.569884+0000","last_active":"2024-03-29T04:33:09.566175+0000","last_peered":"2024-03-29T04:33:09.566175+0000","last_clean":"2024-03-29T04:33:09.566175+0000","last_became_active":"2024-03-29T04:32:31.569682+0000","last_became_peered":"2024-03-29T04:32:31.569682+0000","last_unstale":"2024-03-29T04:33:09.566175+0000","last_undegraded":"2024-03-29T04:33:09.566175+0000","last_fullsized":"2024-03-29T04:33:09.566175+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:49:57.154622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566643+0000","last_change":"2024-03-29T04:32:31.599699+0000","last_active":"2024-03-29T04:33:09.566643+0000","last_peered":"2024-03-29T04:33:09.566643+0000","last_clean":"2024-03-29T04:33:09.566643+0000","last_became_active":"2024-03-29T04:32:31.599298+0000","last_became_peered":"2024-03-29T04:32:31.599298+0000","last_unstale":"2024-03-29T04:33:09.566643+0000","last_undegraded":"2024-03-29T04:33:09.566643+0000","last_fullsized":"2024-03-29T04:33:09.566643+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:16:52.575247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566907+0000","last_change":"2024-03-29T04:32:31.567585+0000","last_active":"2024-03-29T04:33:09.566907+0000","last_peered":"2024-03-29T04:33:09.566907+0000","last_clean":"2024-03-29T04:33:09.566907+0000","last_became_active":"2024-03-29T04:32:31.567306+0000","last_became_peered":"2024-03-29T04:32:31.567306+0000","last_unstale":"2024-03-29T04:33:09.566907+0000","last_undegraded":"2024-03-29T04:33:09.566907+0000","last_fullsized":"2024-03-29T04:33:09.566907+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:48:36.533604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567181+0000","last_change":"2024-03-29T04:32:31.597871+0000","last_active":"2024-03-29T04:33:09.567181+0000","last_peered":"2024-03-29T04:33:09.567181+0000","last_clean":"2024-03-29T04:33:09.567181+0000","last_became_active":"2024-03-29T04:32:31.597588+0000","last_became_peered":"2024-03-29T04:32:31.597588+0000","last_unstale":"2024-03-29T04:33:09.567181+0000","last_undegraded":"2024-03-29T04:33:09.567181+0000","last_fullsized":"2024-03-29T04:33:09.567181+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:56:24.378566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566029+0000","last_change":"2024-03-29T04:32:31.582734+0000","last_active":"2024-03-29T04:33:09.566029+0000","last_peered":"2024-03-29T04:33:09.566029+0000","last_clean":"2024-03-29T04:33:09.566029+0000","last_became_active":"2024-03-29T04:32:31.581340+0000","last_became_peered":"2024-03-29T04:32:31.581340+0000","last_unstale":"2024-03-29T04:33:09.566029+0000","last_undegraded":"2024-03-29T04:33:09.566029+0000","last_fullsized":"2024-03-29T04:33:09.566029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:26:54.916930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560878+0000","last_change":"2024-03-29T04:32:31.773690+0000","last_active":"2024-03-29T04:33:09.560878+0000","last_peered":"2024-03-29T04:33:09.560878+0000","last_clean":"2024-03-29T04:33:09.560878+0000","last_became_active":"2024-03-29T04:32:31.773395+0000","last_became_peered":"2024-03-29T04:32:31.773395+0000","last_unstale":"2024-03-29T04:33:09.560878+0000","last_undegraded":"2024-03-29T04:33:09.560878+0000","last_fullsized":"2024-03-29T04:33:09.560878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:12:59.816689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560863+0000","last_change":"2024-03-29T04:32:31.572581+0000","last_active":"2024-03-29T04:33:09.560863+0000","last_peered":"2024-03-29T04:33:09.560863+0000","last_clean":"2024-03-29T04:33:09.560863+0000","last_became_active":"2024-03-29T04:32:31.572271+0000","last_became_peered":"2024-03-29T04:32:31.572271+0000","last_unstale":"2024-03-29T04:33:09.560863+0000","last_undegraded":"2024-03-29T04:33:09.560863+0000","last_fullsized":"2024-03-29T04:33:09.560863+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:50:10.131117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.560417+0000","last_change":"2024-03-29T04:32:31.593572+0000","last_active":"2024-03-29T04:33:09.560417+0000","last_peered":"2024-03-29T04:33:09.560417+0000","last_clean":"2024-03-29T04:33:09.560417+0000","last_became_active":"2024-03-29T04:32:31.593256+0000","last_became_peered":"2024-03-29T04:32:31.593256+0000","last_unstale":"2024-03-29T04:33:09.560417+0000","last_undegraded":"2024-03-29T04:33:09.560417+0000","last_fullsized":"2024-03-29T04:33:09.560417+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:16:03.370981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056692+0000","last_change":"2024-03-29T04:32:31.577919+0000","last_active":"2024-03-29T04:33:10.056692+0000","last_peered":"2024-03-29T04:33:10.056692+0000","last_clean":"2024-03-29T04:33:10.056692+0000","last_became_active":"2024-03-29T04:32:31.577490+0000","last_became_peered":"2024-03-29T04:32:31.577490+0000","last_unstale":"2024-03-29T04:33:10.056692+0000","last_undegraded":"2024-03-29T04:33:10.056692+0000","last_fullsized":"2024-03-29T04:33:10.056692+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:32:51.541477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.568722+0000","last_change":"2024-03-29T04:32:31.599080+0000","last_active":"2024-03-29T04:33:09.568722+0000","last_peered":"2024-03-29T04:33:09.568722+0000","last_clean":"2024-03-29T04:33:09.568722+0000","last_became_active":"2024-03-29T04:32:31.598739+0000","last_became_peered":"2024-03-29T04:32:31.598739+0000","last_unstale":"2024-03-29T04:33:09.568722+0000","last_undegraded":"2024-03-29T04:33:09.568722+0000","last_fullsized":"2024-03-29T04:33:09.568722+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:08:32.915193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.561872+0000","last_change":"2024-03-29T04:32:31.572300+0000","last_active":"2024-03-29T04:33:09.561872+0000","last_peered":"2024-03-29T04:33:09.561872+0000","last_clean":"2024-03-29T04:33:09.561872+0000","last_became_active":"2024-03-29T04:32:31.571969+0000","last_became_peered":"2024-03-29T04:32:31.571969+0000","last_unstale":"2024-03-29T04:33:09.561872+0000","last_undegraded":"2024-03-29T04:33:09.561872+0000","last_fullsized":"2024-03-29T04:33:09.561872+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:34:49.039328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:10.056925+0000","last_change":"2024-03-29T04:32:31.773604+0000","last_active":"2024-03-29T04:33:10.056925+0000","last_peered":"2024-03-29T04:33:10.056925+0000","last_clean":"2024-03-29T04:33:10.056925+0000","last_became_active":"2024-03-29T04:32:31.773333+0000","last_became_peered":"2024-03-29T04:32:31.773333+0000","last_unstale":"2024-03-29T04:33:10.056925+0000","last_undegraded":"2024-03-29T04:33:10.056925+0000","last_fullsized":"2024-03-29T04:33:10.056925+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:19:59.776976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.567613+0000","last_change":"2024-03-29T04:32:31.572883+0000","last_active":"2024-03-29T04:33:09.567613+0000","last_peered":"2024-03-29T04:33:09.567613+0000","last_clean":"2024-03-29T04:33:09.567613+0000","last_became_active":"2024-03-29T04:32:31.572588+0000","last_became_peered":"2024-03-29T04:32:31.572588+0000","last_unstale":"2024-03-29T04:33:09.567613+0000","last_undegraded":"2024-03-29T04:33:09.567613+0000","last_fullsized":"2024-03-29T04:33:09.567613+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:48:51.961446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.563219+0000","last_change":"2024-03-29T04:32:31.594266+0000","last_active":"2024-03-29T04:33:09.563219+0000","last_peered":"2024-03-29T04:33:09.563219+0000","last_clean":"2024-03-29T04:33:09.563219+0000","last_became_active":"2024-03-29T04:32:31.593992+0000","last_became_peered":"2024-03-29T04:32:31.593992+0000","last_unstale":"2024-03-29T04:33:09.563219+0000","last_undegraded":"2024-03-29T04:33:09.563219+0000","last_fullsized":"2024-03-29T04:33:09.563219+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:46:08.902905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-03-29T04:33:09.566419+0000","last_change":"2024-03-29T04:32:31.571035+0000","last_active":"2024-03-29T04:33:09.566419+0000","last_peered":"2024-03-29T04:33:09.566419+0000","last_clean":"2024-03-29T04:33:09.566419+0000","last_became_active":"2024-03-29T04:32:31.570708+0000","last_became_peered":"2024-03-29T04:32:31.570708+0000","last_unstale":"2024-03-29T04:33:09.566419+0000","last_undegraded":"2024-03-29T04:33:09.566419+0000","last_fullsized":"2024-03-29T04:33:09.566419+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:08:54.298826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201319,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1300,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1331200,"data_stored":901764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.39700000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.5},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68400000000000005,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.48699999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.53600000000000003,"15min":0.52600000000000002},"min":{"1min":0.42899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.56499999999999995}]},{"osd":2,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.36899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.71799999999999997,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.55900000000000005,"15min":0.55600000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.54100000000000004}]},{"osd":3,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.54300000000000004,"15min":0.53700000000000003},"min":{"1min":0.42899999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.55700000000000005,"15min":0.55100000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.70899999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.60599999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.443,"15min":0.434},"min":{"1min":0.34599999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.436},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.442,"15min":0.435},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.51600000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.45700000000000002,"15min":0.45100000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.502,"15min":0.496},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.53000000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:32:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50800000000000001,"15min":0.502},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.57899999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462957,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298648,"kb_used_data":1288,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446504,"statfs":{"total":95995035648,"available":95689220096,"internally_reserved":0,"allocated":1318912,"data_stored":901676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.53500000000000003,"15min":0.52100000000000002},"min":{"1min":0.39600000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72599999999999998,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.41199999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.77100000000000002,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.47199999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54900000000000004,"15min":0.55700000000000005},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.77000000000000002,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.55000000000000004,"15min":0.54300000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71899999999999997,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.46200000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.42799999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.75900000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.53100000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.54000000000000004,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58199999999999996,"15min":0.57699999999999996},"min":{"1min":0.45900000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84599999999999997,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.51700000000000002}]},{"osd":4,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46600000000000003,"15min":0.45500000000000002},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.46999999999999997,"15min":0.46600000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.38500000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:33:16 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.498,"15min":0.48299999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.376},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.504,"15min":0.48799999999999999},"min":{"1min":0.34599999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.441}]},{"osd":7,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56100000000000005,"15min":0.56200000000000006},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.746,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.42799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574917,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":704512,"data_stored":309421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36899999999999999,"15min":0.33700000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.316},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.375,"15min":0.34200000000000003},"min":{"1min":0.24099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.502,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.33900000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.40200000000000002,"15min":0.38400000000000001},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.45100000000000001,"15min":0.441},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.61099999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.39000000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.41799999999999998,"15min":0.436},"min":{"1min":0.27400000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45500000000000002,"15min":0.46700000000000003},"min":{"1min":0.32800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.36299999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.57899999999999996,"15min":0.57099999999999995},"min":{"1min":0.45200000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80400000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56100000000000005,"15min":0.55600000000000005},"min":{"1min":0.41499999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.745,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.57599999999999996}]},{"osd":5,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.60599999999999998,"15min":0.60999999999999999},"min":{"1min":0.47599999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.62,"5min":0.59099999999999997,"15min":0.59799999999999998},"min":{"1min":0.39200000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.67100000000000004}]},{"osd":6,"last update":"Fri Mar 29 04:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.56599999999999995,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80500000000000005,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.60799999999999998,"15min":0.59599999999999997},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89800000000000002,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.60399999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.62,"15min":0.61699999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.622},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.55900000000000005,"15min":0.54700000000000004},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.64400000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738443,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1280,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1310720,"data_stored":900646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36499999999999999,"15min":0.33900000000000002},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.41399999999999998,"15min":0.36499999999999999},"min":{"1min":0.28999999999999998,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.35499999999999998}]},{"osd":2,"last update":"Fri Mar 29 04:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40400000000000003,"15min":0.40799999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.33600000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.47399999999999998,"15min":0.48699999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.309},{"interface":"front","average":{"1min":0.433,"5min":0.45900000000000002,"15min":0.47399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.375}]},{"osd":4,"last update":"Fri Mar 29 04:33:30 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.60799999999999998,"15min":0.59899999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.80900000000000005,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.41799999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.81299999999999994,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.53000000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:32:50 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.59199999999999997,"15min":0.59099999999999997},"min":{"1min":0.437,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.85499999999999998,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64400000000000002,"15min":0.65900000000000003},"min":{"1min":0.42099999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.63600000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.622,"15min":0.628},"min":{"1min":0.44,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76600000000000001,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.60499999999999998,"15min":0.59699999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.86099999999999999,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.752}]},{"osd":7,"last update":"Fri Mar 29 04:32:41 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.61799999999999999,"15min":0.61199999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.60399999999999998,"15min":0.59799999999999998},"min":{"1min":0.495,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.57299999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411392,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298052,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447100,"statfs":{"total":95995035648,"available":95689830400,"internally_reserved":0,"allocated":712704,"data_stored":309684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.377,"15min":0.36799999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52600000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40200000000000002,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.45400000000000001,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.35199999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.42299999999999999,"15min":0.41799999999999998},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.504,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.47399999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.33800000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.432,"15min":0.46999999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.55000000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.495,"15min":0.52500000000000002},"min":{"1min":0.35999999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.64800000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.36399999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59799999999999998,"15min":0.61199999999999999},"min":{"1min":0.433,"5min":0.376,"15min":0.376},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65000000000000002,"15min":0.66100000000000003},"min":{"1min":0.495,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.48599999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:32:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61099999999999999,"15min":0.625},"min":{"1min":0.45400000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.70099999999999996,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56899999999999995,"15min":0.57899999999999996},"min":{"1min":0.432,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.66200000000000003,"5min":0.75,"15min":0.75},"last":0.438}]},{"osd":6,"last update":"Fri Mar 29 04:33:18 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44800000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.85299999999999998,"5min":1.01,"15min":1.01},"last":0.44700000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:32:42 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.59999999999999998,"15min":0.59799999999999998},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.5},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.495,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.41899999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149755,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.38400000000000001,"15min":0.39400000000000002},"min":{"1min":0.249,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.63200000000000001,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.245},{"interface":"front","average":{"1min":0.377,"5min":0.39900000000000002,"15min":0.40200000000000002},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55800000000000005,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45100000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.41899999999999998,"15min":0.42899999999999999},"min":{"1min":0.26600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.61399999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.373,"5min":0.41099999999999998,"15min":0.42099999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.58199999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.42799999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:33:09 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.435,"15min":0.44500000000000001},"min":{"1min":0.29699999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.57699999999999996,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.42999999999999999,"15min":0.434},"min":{"1min":0.29099999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56399999999999995,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.32200000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.45700000000000002,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70199999999999996,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.44600000000000001,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72099999999999997,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.497}]},{"osd":5,"last update":"Fri Mar 29 04:32:48 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.58499999999999996,"15min":0.58999999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.439,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.64600000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.61599999999999999,"15min":0.627},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.878,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59599999999999997,"15min":0.59399999999999997},"min":{"1min":0.432,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83199999999999996,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.442}]},{"osd":7,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41199999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56999999999999995,"15min":0.56899999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81100000000000005,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.46999999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986230,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298072,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447080,"statfs":{"total":95995035648,"available":95689809920,"internally_reserved":0,"allocated":733184,"data_stored":311384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.52700000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72199999999999998,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.52500000000000002,"15min":0.501},"min":{"1min":0.39400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.69899999999999995,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.50800000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.52500000000000002,"15min":0.51200000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68899999999999995,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53400000000000003,"15min":0.51800000000000002},"min":{"1min":0.35699999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73099999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.49099999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.52300000000000002},"min":{"1min":0.34699999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65700000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54700000000000004,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.72099999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.54700000000000004,"15min":0.53700000000000003},"min":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71599999999999997,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55100000000000005,"15min":0.53300000000000003},"min":{"1min":0.434,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76200000000000001,"5min":0.874,"15min":0.874},"last":0.60399999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.438,"15min":0.41399999999999998},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69499999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.753},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.47899999999999998,"15min":0.44700000000000001},"min":{"1min":0.374,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.64700000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:33:19 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.44500000000000001,"15min":0.41899999999999998},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65600000000000003,"5min":0.745,"15min":0.745},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.45300000000000001,"15min":0.43099999999999999},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.67300000000000004}]},{"osd":7,"last update":"Fri Mar 29 04:32:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.52400000000000002,"15min":0.51800000000000002},"min":{"1min":0.38500000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.52700000000000002,"15min":0.51700000000000002},"min":{"1min":0.34399999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.79700000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724593,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":310268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73199999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.504,"5min":0.52700000000000002,"15min":0.53100000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.628,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.45800000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.67400000000000004,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.41199999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.75600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.499}]},{"osd":2,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.56799999999999995,"15min":0.57499999999999996},"min":{"1min":0.38700000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81200000000000006,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.50700000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39700000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.64600000000000002,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.378,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78400000000000003,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.47699999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.434,"15min":0.41599999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60999999999999999,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.38},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.44900000000000001,"15min":0.41999999999999998},"min":{"1min":0.34699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.64600000000000002,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.32600000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:33:13 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.46899999999999997,"15min":0.44500000000000001},"min":{"1min":0.317,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71299999999999997,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.50700000000000001,"15min":0.47799999999999998},"min":{"1min":0.34899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.69699999999999995,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.36099999999999999}]},{"osd":7,"last update":"Fri Mar 29 04:32:40 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.54300000000000004,"15min":0.53500000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.31}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:33:38.470 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:33:38.470 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:33:38.470 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:33:38.471 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:33:38.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:38 smithi029 ceph-mon[30044]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:38.801 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:38 smithi029 ceph-mon[30044]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:33:38.801 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:38 smithi029 ceph-mon[41899]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:38.801 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:38 smithi029 ceph-mon[41899]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:33:39.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:38 smithi099 ceph-mon[33865]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:39.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:38 smithi099 ceph-mon[33865]: from='client.14775 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:33:39.292 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:40.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:40.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:40.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:40.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:40 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:40.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:40.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:40.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:40.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:40.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:40 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:41.242 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:33:41.242 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-29T04:33:41.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3306280823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:33:41.575 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:41 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3306280823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:33:41.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:41 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3306280823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:33:41.947 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-29T04:33:41.947 INFO:teuthology.run_tasks:Running task print... 2024-03-29T04:33:41.957 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-03-29T04:33:41.958 INFO:teuthology.run_tasks:Running task parallel... 2024-03-29T04:33:41.968 INFO:teuthology.task.parallel:starting parallel... 2024-03-29T04:33:41.969 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-29T04:33:41.969 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-03-29T04:33:41.970 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-03-29T04:33:41.971 INFO:tasks.radosbench:Beginning radosbench... 2024-03-29T04:33:41.971 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-03-29T04:33:41.972 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool create unique_pool_0 16 2024-03-29T04:33:41.972 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-29T04:33:41.972 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-29T04:33:41.973 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi029.front.sepia.ceph.com 2024-03-29T04:33:41.973 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-03-29T04:33:42.250 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:42.255 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:42.537 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:42 smithi029 ceph-mon[30044]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:42.537 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:42 smithi029 ceph-mon[41899]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:42.537 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:42 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:42] "GET /metrics HTTP/1.1" 200 239213 "" "Prometheus/2.43.0" 2024-03-29T04:33:42.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:42 smithi099 ceph-mon[33865]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:44.603 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:44 smithi099 ceph-mon[33865]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:44.604 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:33:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:33:44.832 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:44 smithi029 ceph-mon[30044]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:44.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:44 smithi029 ceph-mon[41899]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:45.627 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2971127970' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-03-29T04:33:45.628 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_0' created 2024-03-29T04:33:45.895 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2971127970' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-03-29T04:33:45.895 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:33:45] ENGINE Bus STOPPING 2024-03-29T04:33:45.896 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:33:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:33:45.896 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:33:45] ENGINE Bus STOPPED 2024-03-29T04:33:45.896 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:33:45] ENGINE Bus STARTING 2024-03-29T04:33:45.896 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:33:45] ENGINE Serving on http://:::9283 2024-03-29T04:33:45.896 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:45 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:33:45] ENGINE Bus STARTED 2024-03-29T04:33:46.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:45 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2971127970' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-03-29T04:33:46.580 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-29T04:33:46.923 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2971127970' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: from='client.? ' entity='client.admin' 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[30044]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2971127970' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-03-29T04:33:46.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:33:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: from='client.? ' entity='client.admin' 2024-03-29T04:33:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:46.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:46 smithi029 ceph-mon[41899]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:33:47.043 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-03-29T04:33:47.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2971127970' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: from='client.? ' entity='client.admin' 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:47.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:46 smithi099 ceph-mon[33865]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:33:47.186 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:47.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:47 smithi029 ceph-mon[30044]: pgmap v94: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:47.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:47 smithi029 ceph-mon[30044]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:33:47.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:47 smithi029 ceph-mon[30044]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:33:47.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:47 smithi029 ceph-mon[41899]: pgmap v94: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:47.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:47 smithi029 ceph-mon[41899]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:33:47.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:47 smithi029 ceph-mon[41899]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:33:47.943 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:48.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:47 smithi099 ceph-mon[33865]: pgmap v94: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:48.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:47 smithi099 ceph-mon[33865]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:33:48.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:47 smithi099 ceph-mon[33865]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:33:50.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:50 smithi029 ceph-mon[30044]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:50.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:50 smithi029 ceph-mon[41899]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:50.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:50 smithi099 ceph-mon[33865]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:51.182 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-29T04:33:51.542 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:52.423 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:33:52 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:52] "GET /metrics HTTP/1.1" 200 239213 "" "Prometheus/2.43.0" 2024-03-29T04:33:52.497 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-03-29T04:33:52.854 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:52 smithi029 ceph-mon[30044]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:52.854 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:52 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/73306630' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:33:52.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:52 smithi099 ceph-mon[33865]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:52.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:52 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/73306630' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:33:52.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:52 smithi029 ceph-mon[41899]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:52.855 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:52 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/73306630' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:33:53.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:53 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/73306630' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:33:53.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:53 smithi029 ceph-mon[30044]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:33:53.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:53 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/73306630' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:33:53.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:53 smithi029 ceph-mon[41899]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:33:53.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:53 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/73306630' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:33:53.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:53 smithi099 ceph-mon[33865]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:33:54.290 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-03-29T04:33:54.293 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-03-29T04:33:54.293 INFO:teuthology.orchestra.run:waiting for 3000 2024-03-29T04:33:54.343 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-03-29T04:33:54.343 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-03-29T04:33:54.343 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_87026 2024-03-29T04:33:54.343 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:33:54.343 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-03-29T04:33:54.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:54 smithi099 ceph-mon[33865]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:54.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:54 smithi099 ceph-mon[33865]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:33:54.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:54 smithi099 ceph-mon[33865]: Cluster is now healthy 2024-03-29T04:33:54.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:54 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:54.604 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:33:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:33:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:33:54.673 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[30044]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[30044]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[30044]: Cluster is now healthy 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[41899]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[41899]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[41899]: Cluster is now healthy 2024-03-29T04:33:54.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:54 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:33:54.864 INFO:teuthology.orchestra.run.smithi029.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:55.343 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3953 3937 246.048 246.062 0.00694312 0.0040489 2024-03-29T04:33:55.819 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:56.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[30044]: pgmap v100: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s wr, 529 op/s 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:56.175 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:55 smithi029 ceph-mon[41899]: pgmap v100: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s wr, 529 op/s 2024-03-29T04:33:56.333 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:33:56.344 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 7648 7632 238.483 230.938 0.00645831 0.00418549 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:56.354 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:55 smithi099 ceph-mon[33865]: pgmap v100: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s wr, 529 op/s 2024-03-29T04:33:57.344 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10958 10942 227.884 206.875 0.0056299 0.00438139 2024-03-29T04:33:58.344 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 14021 14005 218.768 191.438 0.00217367 0.0045329 2024-03-29T04:33:58.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:33:58 smithi099 ceph-mon[33865]: pgmap v101: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s wr, 431 op/s 2024-03-29T04:33:58.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:33:58 smithi029 ceph-mon[30044]: pgmap v101: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s wr, 431 op/s 2024-03-29T04:33:58.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:33:58 smithi029 ceph-mon[41899]: pgmap v101: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s wr, 431 op/s 2024-03-29T04:33:59.344 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 16412 16396 204.902 149.438 0.00631586 0.00487381 2024-03-29T04:34:00.344 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 18798 18782 195.605 149.125 0.00924422 0.00510611 2024-03-29T04:34:00.838 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:00 smithi029 ceph-mon[30044]: pgmap v102: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s wr, 423 op/s 2024-03-29T04:34:00.839 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:00 smithi029 ceph-mon[41899]: pgmap v102: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s wr, 423 op/s 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (92s) 21s ago 2m 25.7M - 0.25.0 c8568f914cd2 f5ce6956110e 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi099 *:3000 running (2m) 24s ago 2m 85.7M - 9.4.7 954c08fa6188 75f07d4cfd30 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.gsovzu smithi029 running (73s) 21s ago 73s 12.7M - 17.2.7-789-g4213e17c cf22eb8683be e6b24be4eed9 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi099.jhvkhg smithi099 running (71s) 24s ago 70s 14.4M - 17.2.7-789-g4213e17c cf22eb8683be 2d2ff55bb03a 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi099 *:8443 running (6m) 24s ago 6m 432M - 17.2.7-789-g4213e17c cf22eb8683be 65870fc341f2 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:9283 running (8m) 21s ago 8m 478M - 17.2.7-789-g4213e17c cf22eb8683be a746f2a9d0b9 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (8m) 21s ago 8m 46.1M 2048M 17.2.7-789-g4213e17c cf22eb8683be 585716eda74d 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi099 running (7m) 24s ago 7m 41.3M 2048M 17.2.7-789-g4213e17c cf22eb8683be 48c38660bc12 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (7m) 21s ago 7m 37.6M 2048M 17.2.7-789-g4213e17c cf22eb8683be 0159b319c9a5 2024-03-29T04:34:00.839 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (2m) 21s ago 2m 16.0M - 1.5.0 0da6a335fe13 4ddfb32c84c0 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi099 *:9100 running (119s) 24s ago 119s 7268k - 1.5.0 0da6a335fe13 01b461e23b83 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (6m) 21s ago 6m 42.8M 4017M 17.2.7-789-g4213e17c cf22eb8683be 5e5c90bac829 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (6m) 21s ago 5m 42.8M 4017M 17.2.7-789-g4213e17c cf22eb8683be 6532f8ba40e9 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (5m) 21s ago 5m 43.6M 4017M 17.2.7-789-g4213e17c cf22eb8683be 817a836d7edf 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (5m) 21s ago 5m 41.0M 4017M 17.2.7-789-g4213e17c cf22eb8683be a4cb24ddb406 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi099 running (4m) 24s ago 4m 42.9M 4016M 17.2.7-789-g4213e17c cf22eb8683be 1fab66997673 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi099 running (4m) 24s ago 4m 43.6M 4016M 17.2.7-789-g4213e17c cf22eb8683be 78420bf6aaa9 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi099 running (3m) 24s ago 3m 40.4M 4016M 17.2.7-789-g4213e17c cf22eb8683be 1f39d06fae80 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi099 running (3m) 24s ago 3m 40.9M 4016M 17.2.7-789-g4213e17c cf22eb8683be 6adf0209fbb4 2024-03-29T04:34:00.840 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi099 *:9095 running (85s) 24s ago 2m 42.0M - 2.43.0 a07b618ecd1d a9fb36d03923 2024-03-29T04:34:00.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:00 smithi099 ceph-mon[33865]: pgmap v102: 89 pgs: 89 active+clean; 265 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s wr, 423 op/s 2024-03-29T04:34:01.344 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 21157 21141 188.723 147.438 0.00536778 0.00529262 2024-03-29T04:34:01.743 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for minority of mons to upgrade' 2024-03-29T04:34:02.240 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:34:02.345 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 23560 23544 183.898 150.188 0.00688069 0.00543279 2024-03-29T04:34:02.387 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:34:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:02] "GET /metrics HTTP/1.1" 200 241550 "" "Prometheus/2.43.0" 2024-03-29T04:34:02.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:02 smithi029 ceph-mon[30044]: from='client.24637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:02.674 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:02 smithi029 ceph-mon[30044]: pgmap v103: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-03-29T04:34:02.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:02 smithi029 ceph-mon[41899]: from='client.24637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:02.674 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:02 smithi029 ceph-mon[41899]: pgmap v103: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-03-29T04:34:02.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:02 smithi099 ceph-mon[33865]: from='client.24637 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:02.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:02 smithi099 ceph-mon[33865]: pgmap v103: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-03-29T04:34:03.346 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 25984 25968 180.279 151.5 0.00717338 0.00554155 2024-03-29T04:34:04.346 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 27929 27913 174.408 121.562 0.00437681 0.00568885 2024-03-29T04:34:04.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:04 smithi099 ceph-mon[33865]: pgmap v104: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-03-29T04:34:04.604 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:34:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:34:04.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:04 smithi029 ceph-mon[41899]: pgmap v104: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-03-29T04:34:04.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:04 smithi029 ceph-mon[30044]: pgmap v104: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-03-29T04:34:05.346 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 29829 29813 169.347 118.75 0.00356312 0.00584041 2024-03-29T04:34:06.346 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 31145 31129 162.089 82.25 0.00201879 0.0060615 2024-03-29T04:34:06.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:06 smithi029 ceph-mon[30044]: pgmap v105: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-03-29T04:34:06.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:06 smithi029 ceph-mon[41899]: pgmap v105: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-03-29T04:34:06.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:06 smithi099 ceph-mon[33865]: pgmap v105: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-03-29T04:34:07.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 32615 32599 156.688 91.875 0.001737 0.00627816 2024-03-29T04:34:08.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 33677 33661 150.236 66.375 0.00474796 0.0066505 2024-03-29T04:34:08.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:08 smithi099 ceph-mon[33865]: pgmap v106: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-03-29T04:34:08.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:08 smithi029 ceph-mon[41899]: pgmap v106: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-03-29T04:34:08.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:08 smithi029 ceph-mon[30044]: pgmap v106: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-03-29T04:34:09.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 34775 34759 144.796 68.625 0.00546954 0.00690095 2024-03-29T04:34:09.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:09.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:09.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:10.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 36283 36267 141.636 94.25 0.00178997 0.00700375 2024-03-29T04:34:10.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:10 smithi099 ceph-mon[33865]: pgmap v107: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-03-29T04:34:10.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:10 smithi029 ceph-mon[30044]: pgmap v107: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-03-29T04:34:10.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:10 smithi029 ceph-mon[41899]: pgmap v107: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2024-03-29T04:34:11.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 37329 37313 137.151 65.375 0.00260229 0.00719106 2024-03-29T04:34:12.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 38900 38884 134.986 98.1875 0.00434242 0.00740306 2024-03-29T04:34:12.525 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:34:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:12] "GET /metrics HTTP/1.1" 200 242490 "" "Prometheus/2.43.0" 2024-03-29T04:34:12.561 INFO:teuthology.orchestra.run.smithi029.stdout:wait for minority of mons to upgrade 2024-03-29T04:34:12.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:12 smithi099 ceph-mon[33865]: pgmap v108: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-03-29T04:34:12.924 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:12 smithi029 ceph-mon[30044]: pgmap v108: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-03-29T04:34:12.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:12 smithi029 ceph-mon[41899]: pgmap v108: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-03-29T04:34:13.347 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 15 40450 40435 132.983 96.9375 0.00803866 0.00751481 2024-03-29T04:34:14.348 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:34:14.366475+0000 min lat: 0.00119188 max lat: 0.471326 avg lat: 0.00770436 2024-03-29T04:34:14.348 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:34:14.348 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 41531 41515 129.709 67.5 0.00554776 0.00770436 2024-03-29T04:34:14.604 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:14 smithi099 ceph-mon[33865]: pgmap v109: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-03-29T04:34:14.605 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:34:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:34:14.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:14 smithi029 ceph-mon[30044]: pgmap v109: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-03-29T04:34:14.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:14 smithi029 ceph-mon[41899]: pgmap v109: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-03-29T04:34:15.349 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 42704 42688 127.016 73.3125 0.0133496 0.00786723 2024-03-29T04:34:16.349 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 43915 43899 124.683 75.6875 0.00584438 0.00801509 2024-03-29T04:34:16.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:16 smithi099 ceph-mon[33865]: pgmap v110: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-03-29T04:34:16.924 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:16 smithi029 ceph-mon[41899]: pgmap v110: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-03-29T04:34:16.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:16 smithi029 ceph-mon[30044]: pgmap v110: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-03-29T04:34:17.349 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 44865 44849 121.844 59.375 0.00732365 0.00820047 2024-03-29T04:34:18.349 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 46215 46199 120.283 84.375 0.00249026 0.00828092 2024-03-29T04:34:18.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:18 smithi099 ceph-mon[33865]: pgmap v111: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-03-29T04:34:18.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:18 smithi029 ceph-mon[30044]: pgmap v111: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-03-29T04:34:18.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:18 smithi029 ceph-mon[41899]: pgmap v111: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-03-29T04:34:19.349 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 46747 46731 116.802 33.25 0.00941805 0.00855704 2024-03-29T04:34:20.350 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 47905 47889 115.093 72.375 0.0091142 0.00868355 2024-03-29T04:34:20.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:20 smithi099 ceph-mon[33865]: pgmap v112: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-03-29T04:34:20.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:20 smithi029 ceph-mon[30044]: pgmap v112: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-03-29T04:34:20.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:20 smithi029 ceph-mon[41899]: pgmap v112: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-03-29T04:34:21.350 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 49448 49432 114.4 96.4375 0.0122063 0.00873564 2024-03-29T04:34:22.350 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 50800 50784 113.332 84.5 0.0198828 0.00881873 2024-03-29T04:34:22.365 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:34:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:22] "GET /metrics HTTP/1.1" 200 242490 "" "Prometheus/2.43.0" 2024-03-29T04:34:22.675 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:22 smithi029 ceph-mon[30044]: pgmap v113: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-03-29T04:34:22.675 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:22 smithi029 ceph-mon[41899]: pgmap v113: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-03-29T04:34:22.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:22 smithi099 ceph-mon[33865]: pgmap v113: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-03-29T04:34:23.350 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 51815 51799 111.612 63.4375 0.00508593 0.0089533 2024-03-29T04:34:24.350 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 53528 53512 111.46 107.062 0.0117093 0.00896706 2024-03-29T04:34:24.605 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:24 smithi099 ceph-mon[33865]: pgmap v114: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2024-03-29T04:34:24.605 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:24 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:24.605 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:34:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:34:24.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:24 smithi029 ceph-mon[30044]: pgmap v114: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2024-03-29T04:34:24.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:24 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:24.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:24 smithi029 ceph-mon[41899]: pgmap v114: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2024-03-29T04:34:24.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:24 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:25.350 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 55160 55144 111.154 102 0.0124867 0.00899209 2024-03-29T04:34:26.351 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 56759 56743 110.804 99.9375 0.00736236 0.00902012 2024-03-29T04:34:26.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:26 smithi099 ceph-mon[33865]: pgmap v115: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-03-29T04:34:26.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:26 smithi029 ceph-mon[30044]: pgmap v115: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-03-29T04:34:26.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:26 smithi029 ceph-mon[41899]: pgmap v115: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-03-29T04:34:27.351 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 58420 58404 110.592 103.812 0.0122885 0.00903712 2024-03-29T04:34:28.352 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 60047 60031 110.327 101.688 0.00694126 0.0090583 2024-03-29T04:34:28.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:28 smithi099 ceph-mon[33865]: pgmap v116: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-03-29T04:34:28.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:28 smithi029 ceph-mon[30044]: pgmap v116: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-03-29T04:34:28.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:28 smithi029 ceph-mon[41899]: pgmap v116: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-03-29T04:34:29.352 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 61535 61519 109.832 93 0.00589358 0.00909954 2024-03-29T04:34:30.352 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 63061 63045 109.43 95.375 0.0108702 0.00913393 2024-03-29T04:34:30.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:30 smithi099 ceph-mon[33865]: pgmap v117: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-03-29T04:34:30.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:30 smithi029 ceph-mon[30044]: pgmap v117: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-03-29T04:34:30.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:30 smithi029 ceph-mon[41899]: pgmap v117: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-03-29T04:34:31.352 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 64572 64556 109.025 94.4375 0.0131036 0.00916768 2024-03-29T04:34:32.352 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 66042 66026 108.573 91.875 0.00812539 0.0092054 2024-03-29T04:34:32.569 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:34:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:32] "GET /metrics HTTP/1.1" 200 242619 "" "Prometheus/2.43.0" 2024-03-29T04:34:32.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:32 smithi099 ceph-mon[33865]: pgmap v118: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-03-29T04:34:32.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:32 smithi029 ceph-mon[30044]: pgmap v118: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-03-29T04:34:32.925 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:32 smithi029 ceph-mon[41899]: pgmap v118: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-03-29T04:34:33.352 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 67327 67311 107.848 80.3125 0.0113092 0.00926725 2024-03-29T04:34:34.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:34:34.370571+0000 min lat: 0.00119188 max lat: 0.670349 avg lat: 0.00927947 2024-03-29T04:34:34.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:34:34.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 68740 68724 107.36 88.3125 0.00365985 0.00927947 2024-03-29T04:34:34.573 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:34:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:34:34.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:34 smithi099 ceph-mon[33865]: pgmap v119: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-03-29T04:34:34.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:34 smithi029 ceph-mon[30044]: pgmap v119: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-03-29T04:34:34.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:34 smithi029 ceph-mon[41899]: pgmap v119: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-03-29T04:34:35.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 69849 69833 106.432 69.3125 0.00282164 0.00938087 2024-03-29T04:34:36.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 71225 71209 105.945 86 0.00897071 0.00943425 2024-03-29T04:34:36.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:36 smithi099 ceph-mon[33865]: pgmap v120: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:34:36.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:36 smithi029 ceph-mon[30044]: pgmap v120: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:34:36.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:36 smithi029 ceph-mon[41899]: pgmap v120: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:34:37.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 72637 72621 105.533 88.25 0.00857079 0.00947045 2024-03-29T04:34:38.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 74063 74047 105.16 89.125 0.00836458 0.00950471 2024-03-29T04:34:38.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:38 smithi099 ceph-mon[33865]: pgmap v121: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-03-29T04:34:38.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:38 smithi029 ceph-mon[30044]: pgmap v121: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-03-29T04:34:38.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:38 smithi029 ceph-mon[41899]: pgmap v121: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-03-29T04:34:39.353 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 75281 75265 104.515 76.125 0.109391 0.00955478 2024-03-29T04:34:39.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:39 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:39.925 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:39 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:39.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:39 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:40.179 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-03-29T04:34:40.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 76748 76732 104.234 91.6875 0.00967063 0.00958914 2024-03-29T04:34:40.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:40 smithi099 ceph-mon[33865]: pgmap v122: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-03-29T04:34:40.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:40 smithi029 ceph-mon[41899]: pgmap v122: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-03-29T04:34:40.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:40 smithi029 ceph-mon[30044]: pgmap v122: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-03-29T04:34:41.354 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 77833 77817 103.459 67.8125 0.00690483 0.0096608 2024-03-29T04:34:42.266 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:34:42.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 79271 79255 103.175 89.875 0.00548867 0.00968754 2024-03-29T04:34:42.426 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:34:42 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:42] "GET /metrics HTTP/1.1" 200 242676 "" "Prometheus/2.43.0" 2024-03-29T04:34:42.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:42 smithi099 ceph-mon[33865]: pgmap v123: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-03-29T04:34:42.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:42 smithi029 ceph-mon[30044]: pgmap v123: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-03-29T04:34:42.858 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:42 smithi029 ceph-mon[41899]: pgmap v123: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-03-29T04:34:43.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 80722 80706 102.92 90.6875 0.01001 0.00971155 2024-03-29T04:34:44.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 82091 82075 102.573 85.5625 0.00807472 0.00974393 2024-03-29T04:34:44.597 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:34:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:34:44.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:44 smithi099 ceph-mon[33865]: pgmap v124: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-03-29T04:34:44.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:34:44.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:44 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:44.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:44 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:34:44.890 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[30044]: pgmap v124: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[41899]: pgmap v124: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:44.891 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:44 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:34:45.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 83365 83349 102.123 79.625 0.0105821 0.00978723 2024-03-29T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:45 smithi099 ceph-mon[33865]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:45 smithi099 ceph-mon[33865]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:45 smithi099 ceph-mon[33865]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:45 smithi099 ceph-mon[33865]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi099 2024-03-29T04:34:45.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:45 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3169266836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[30044]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[30044]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[30044]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[30044]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi099 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3169266836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[41899]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:34:45.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[41899]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:34:45.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[41899]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:34:45.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[41899]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi099 2024-03-29T04:34:45.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:45 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3169266836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:46.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 84554 84538 101.588 74.3125 0.00586603 0.00983888 2024-03-29T04:34:46.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:46 smithi099 ceph-mon[33865]: pgmap v125: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-03-29T04:34:46.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:46 smithi029 ceph-mon[30044]: pgmap v125: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-03-29T04:34:46.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:46 smithi029 ceph-mon[41899]: pgmap v125: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-03-29T04:34:47.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 85751 85735 101.083 74.8125 0.0112301 0.00988773 2024-03-29T04:34:48.355 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 86956 86940 100.606 75.3125 0.00603737 0.00993538 2024-03-29T04:34:48.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:48 smithi029 ceph-mon[30044]: pgmap v126: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-03-29T04:34:48.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:48 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1150916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:48.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:48 smithi029 ceph-mon[41899]: pgmap v126: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-03-29T04:34:48.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:48 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1150916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:49.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:48 smithi099 ceph-mon[33865]: pgmap v126: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.37k op/s 2024-03-29T04:34:49.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:48 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1150916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:49.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 15 88197 88182 100.188 77.625 0.0124148 0.00997676 2024-03-29T04:34:50.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 89285 89269 99.6118 67.9375 0.01321 0.0100344 2024-03-29T04:34:50.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:50 smithi029 ceph-mon[30044]: pgmap v127: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-03-29T04:34:50.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:50 smithi029 ceph-mon[41899]: pgmap v127: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-03-29T04:34:51.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:50 smithi099 ceph-mon[33865]: pgmap v127: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-03-29T04:34:51.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 90439 90423 99.1294 72.125 0.00855232 0.010076 2024-03-29T04:34:51.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:51 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1861119873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:51.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:51 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1861119873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:52.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:51 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1861119873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:52.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 91457 91441 98.5172 63.625 0.0177457 0.0101458 2024-03-29T04:34:52.426 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:34:52 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:52] "GET /metrics HTTP/1.1" 200 242676 "" "Prometheus/2.43.0" 2024-03-29T04:34:52.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:52 smithi029 ceph-mon[30044]: pgmap v128: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-03-29T04:34:52.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:52 smithi029 ceph-mon[41899]: pgmap v128: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-03-29T04:34:53.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:52 smithi099 ceph-mon[33865]: pgmap v128: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-03-29T04:34:53.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 92568 92552 98.0243 69.4375 0.00828577 0.0101969 2024-03-29T04:34:53.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:53 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1730738339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:53.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:53 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1730738339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:54.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:53 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1730738339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:54.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:34:54.373525+0000 min lat: 0.00119188 max lat: 0.670349 avg lat: 0.0102429 2024-03-29T04:34:54.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:34:54.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 93712 93696 97.5822 71.5 0.0129632 0.0102429 2024-03-29T04:34:54.606 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:34:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:34:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:34:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:54 smithi029 ceph-mon[30044]: pgmap v129: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-03-29T04:34:54.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:54 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:54.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:54 smithi029 ceph-mon[41899]: pgmap v129: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-03-29T04:34:54.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:54 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:55.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:54 smithi099 ceph-mon[33865]: pgmap v129: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-03-29T04:34:55.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:54 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:34:55.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 94857 94841 97.1555 71.5625 0.0120778 0.0102883 2024-03-29T04:34:56.356 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 95993 95977 96.7336 71 0.00875971 0.0103327 2024-03-29T04:34:56.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:56 smithi029 ceph-mon[30044]: pgmap v130: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:34:56.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:56 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1635336054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:56.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:56 smithi029 ceph-mon[41899]: pgmap v130: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:34:56.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:56 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1635336054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:57.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:56 smithi099 ceph-mon[33865]: pgmap v130: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:34:57.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:56 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1635336054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:57.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 97134 97118 96.33 71.3125 0.00897908 0.0103759 2024-03-29T04:34:58.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 98264 98248 95.9283 70.625 0.0131663 0.0104196 2024-03-29T04:34:58.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:58 smithi029 ceph-mon[30044]: pgmap v131: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-03-29T04:34:58.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:58 smithi029 ceph-mon[41899]: pgmap v131: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-03-29T04:34:59.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:58 smithi099 ceph-mon[33865]: pgmap v131: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-03-29T04:34:59.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 99427 99411 95.5707 72.6875 0.0210481 0.0104587 2024-03-29T04:34:59.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:34:59 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3565337019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:34:59.926 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:34:59 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3565337019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:00.007 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:34:59 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3565337019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:00.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 100529 100513 95.1662 68.875 0.0186813 0.0105026 2024-03-29T04:35:00.926 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:00 smithi029 ceph-mon[30044]: pgmap v132: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-03-29T04:35:00.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:00 smithi029 ceph-mon[41899]: pgmap v132: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-03-29T04:35:01.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:00 smithi099 ceph-mon[33865]: pgmap v132: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-03-29T04:35:01.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 101586 101570 94.7317 66.0625 0.0106035 0.0105515 2024-03-29T04:35:01.690 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:01 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1290878515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:02.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:01 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1290878515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:02.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:01 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1290878515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:02.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 102521 102505 94.198 58.4375 0.0188649 0.0106108 2024-03-29T04:35:02.676 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:02] "GET /metrics HTTP/1.1" 200 242765 "" "Prometheus/2.43.0" 2024-03-29T04:35:02.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:02 smithi099 ceph-mon[33865]: pgmap v133: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-03-29T04:35:03.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:02 smithi029 ceph-mon[30044]: pgmap v133: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-03-29T04:35:03.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:02 smithi029 ceph-mon[41899]: pgmap v133: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-03-29T04:35:03.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 103527 103511 93.744 62.875 0.00699504 0.0106624 2024-03-29T04:35:04.357 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 104574 104558 93.3395 65.4375 0.0180089 0.0107089 2024-03-29T04:35:04.607 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:35:05.107 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:04 smithi099 ceph-mon[33865]: pgmap v134: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-03-29T04:35:05.107 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:04 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2573727214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:05.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:04 smithi029 ceph-mon[30044]: pgmap v134: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-03-29T04:35:05.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:04 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2573727214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:05.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:04 smithi029 ceph-mon[41899]: pgmap v134: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-03-29T04:35:05.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:04 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2573727214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:05.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 105665 105649 92.9852 68.1875 0.0390166 0.0107485 2024-03-29T04:35:06.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:05 smithi099 ceph-mon[33865]: pgmap v135: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-03-29T04:35:06.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:05 smithi029 ceph-mon[30044]: pgmap v135: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-03-29T04:35:06.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:05 smithi029 ceph-mon[41899]: pgmap v135: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-03-29T04:35:06.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 106738 106722 92.6251 67.0625 0.0150375 0.0107912 2024-03-29T04:35:07.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:06 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/882264080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:07.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 107818 107802 92.2809 67.5 0.0165266 0.0108316 2024-03-29T04:35:07.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:06 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/882264080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:07.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:06 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/882264080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:08.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:08 smithi099 ceph-mon[33865]: pgmap v136: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-03-29T04:35:08.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 108887 108871 91.9367 66.8125 0.00708314 0.0108721 2024-03-29T04:35:08.427 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:08 smithi029 ceph-mon[30044]: pgmap v136: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-03-29T04:35:08.427 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:08 smithi029 ceph-mon[41899]: pgmap v136: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-03-29T04:35:09.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 109917 109901 91.5692 64.375 0.00604105 0.0109159 2024-03-29T04:35:09.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:09 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:09.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:09 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:09.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:09 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:10.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 110938 110922 91.2039 63.8125 0.0119879 0.0109598 2024-03-29T04:35:10.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:10 smithi099 ceph-mon[33865]: pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-03-29T04:35:10.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:10 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/299519934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:10.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:10 smithi029 ceph-mon[30044]: pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-03-29T04:35:10.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:10 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/299519934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:10.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:10 smithi029 ceph-mon[41899]: pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-03-29T04:35:10.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:10 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/299519934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:11.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 111971 111955 90.8579 64.5625 0.0208363 0.0110009 2024-03-29T04:35:12.358 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 112965 112949 90.4895 62.125 0.0129125 0.0110456 2024-03-29T04:35:12.538 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:12] "GET /metrics HTTP/1.1" 200 242785 "" "Prometheus/2.43.0" 2024-03-29T04:35:12.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:12 smithi099 ceph-mon[33865]: pgmap v138: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-03-29T04:35:12.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:12 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/221790619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:12.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:12 smithi029 ceph-mon[30044]: pgmap v138: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-03-29T04:35:12.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:12 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/221790619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:12.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:12 smithi029 ceph-mon[41899]: pgmap v138: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-03-29T04:35:12.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:12 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/221790619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:13.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 113970 113954 90.1391 62.8125 0.0192704 0.0110885 2024-03-29T04:35:14.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:35:14.375260+0000 min lat: 0.00119188 max lat: 0.670349 avg lat: 0.0111342 2024-03-29T04:35:14.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:35:14.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 114944 114928 89.7732 60.875 0.0237004 0.0111342 2024-03-29T04:35:14.607 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:14 smithi099 ceph-mon[33865]: pgmap v139: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2024-03-29T04:35:14.607 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:35:14.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:14 smithi029 ceph-mon[30044]: pgmap v139: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2024-03-29T04:35:14.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:14 smithi029 ceph-mon[41899]: pgmap v139: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2024-03-29T04:35:15.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 115947 115931 89.4388 62.6875 0.011047 0.0111763 2024-03-29T04:35:15.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:15 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3419119853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:15.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:15 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3419119853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:15.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:15 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3419119853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:16.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 116957 116941 89.1178 63.125 0.0173277 0.0112162 2024-03-29T04:35:16.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:16 smithi099 ceph-mon[33865]: pgmap v140: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-03-29T04:35:16.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:16 smithi029 ceph-mon[30044]: pgmap v140: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-03-29T04:35:16.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:16 smithi029 ceph-mon[41899]: pgmap v140: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s 2024-03-29T04:35:17.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 117954 117938 88.7948 62.3125 0.0137821 0.0112572 2024-03-29T04:35:18.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 118858 118842 88.4104 56.5 0.00758781 0.0113055 2024-03-29T04:35:18.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:18 smithi099 ceph-mon[33865]: pgmap v141: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-03-29T04:35:18.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:18 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/697556697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:18.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:18 smithi029 ceph-mon[30044]: pgmap v141: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-03-29T04:35:18.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:18 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/697556697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:18.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:18 smithi029 ceph-mon[41899]: pgmap v141: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-03-29T04:35:18.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:18 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/697556697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:19.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 119721 119705 88.0048 53.9375 0.00996005 0.0113574 2024-03-29T04:35:20.359 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 120597 120581 87.618 54.75 0.00701584 0.0114074 2024-03-29T04:35:20.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:20 smithi099 ceph-mon[33865]: pgmap v142: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-03-29T04:35:20.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:20 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1914025831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:20.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:20 smithi029 ceph-mon[30044]: pgmap v142: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-03-29T04:35:20.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:20 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1914025831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:20.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:20 smithi029 ceph-mon[41899]: pgmap v142: 89 pgs: 89 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-03-29T04:35:20.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:20 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1914025831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:21.360 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 121453 121437 87.2259 53.5 0.0119004 0.0114591 2024-03-29T04:35:22.360 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 122311 122295 86.844 53.625 0.020017 0.0115092 2024-03-29T04:35:22.583 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:22] "GET /metrics HTTP/1.1" 200 242785 "" "Prometheus/2.43.0" 2024-03-29T04:35:22.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:22 smithi099 ceph-mon[33865]: pgmap v143: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:35:22.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:22 smithi029 ceph-mon[30044]: pgmap v143: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:35:22.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:22 smithi029 ceph-mon[41899]: pgmap v143: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:35:23.360 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 123173 123157 86.4735 53.875 0.0148974 0.0115591 2024-03-29T04:35:23.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:23 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2385477646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:23.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:23 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2385477646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:23.927 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:23 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2385477646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:24.360 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0064 2024-03-29T04:35:24.360 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 124012 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 86.1133 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 37.1492 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 246.062 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 33.25 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1377 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 594.388 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3937 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 532 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0116086 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0189084 2024-03-29T04:35:24.361 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 0.670349 2024-03-29T04:35:24.362 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00119188 2024-03-29T04:35:24.365 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-03-29T04:35:24.599 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:35:24.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:24 smithi099 ceph-mon[33865]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2024-03-29T04:35:24.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:24 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:24.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:24 smithi029 ceph-mon[30044]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2024-03-29T04:35:24.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:24 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:24.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:24 smithi029 ceph-mon[41899]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2024-03-29T04:35:24.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:24 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:25.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:25 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/856139683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:25.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:25 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/856139683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:25.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:25 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/856139683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:26.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:26 smithi099 ceph-mon[33865]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 864 op/s 2024-03-29T04:35:26.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:26 smithi029 ceph-mon[30044]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 864 op/s 2024-03-29T04:35:26.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:26 smithi029 ceph-mon[41899]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 864 op/s 2024-03-29T04:35:28.927 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:28 smithi029 ceph-mon[30044]: pgmap v146: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.07k op/s 2024-03-29T04:35:28.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:28 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/197811779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:28.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:28 smithi029 ceph-mon[41899]: pgmap v146: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.07k op/s 2024-03-29T04:35:28.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:28 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/197811779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:29.070 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:28 smithi099 ceph-mon[33865]: pgmap v146: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.07k op/s 2024-03-29T04:35:29.071 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:28 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/197811779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[30044]: pgmap v147: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 731 op/s 2024-03-29T04:35:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:35:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:35:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:35:30.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[41899]: pgmap v147: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 731 op/s 2024-03-29T04:35:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:35:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:35:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:35:30.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:30 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:30.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:30 smithi099 ceph-mon[33865]: pgmap v147: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 731 op/s 2024-03-29T04:35:30.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:30 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:30.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:30 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:35:30.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:30 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:35:30.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:30 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:35:30.994 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:30 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:31.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:31 smithi029 ceph-mon[30044]: Upgrade: Updating mgr.x 2024-03-29T04:35:31.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:31 smithi029 ceph-mon[30044]: Deploying daemon mgr.x on smithi099 2024-03-29T04:35:31.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:31 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2209388251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:31.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:31 smithi029 ceph-mon[41899]: Upgrade: Updating mgr.x 2024-03-29T04:35:31.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:31 smithi029 ceph-mon[41899]: Deploying daemon mgr.x on smithi099 2024-03-29T04:35:31.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:31 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2209388251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:32.107 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:31 smithi099 ceph-mon[33865]: Upgrade: Updating mgr.x 2024-03-29T04:35:32.108 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:31 smithi099 ceph-mon[33865]: Deploying daemon mgr.x on smithi099 2024-03-29T04:35:32.108 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:31 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2209388251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:32.626 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:32] "GET /metrics HTTP/1.1" 200 242784 "" "Prometheus/2.43.0" 2024-03-29T04:35:32.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:32 smithi099 ceph-mon[33865]: pgmap v148: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 1.11k op/s 2024-03-29T04:35:32.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:32 smithi029 ceph-mon[30044]: pgmap v148: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 1.11k op/s 2024-03-29T04:35:32.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:32 smithi029 ceph-mon[41899]: pgmap v148: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 1.11k op/s 2024-03-29T04:35:33.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:33 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3411027337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:33.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:33 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3411027337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:33.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:33 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3411027337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:34.608 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[35984]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:35:34.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:34 smithi029 ceph-mon[30044]: pgmap v149: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 729 op/s 2024-03-29T04:35:34.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:34 smithi029 ceph-mon[41899]: pgmap v149: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 729 op/s 2024-03-29T04:35:35.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:34 smithi099 ceph-mon[33865]: pgmap v149: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 729 op/s 2024-03-29T04:35:36.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:36 smithi099 ceph-mon[33865]: pgmap v150: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 776 op/s 2024-03-29T04:35:36.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:36 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3468021338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:36.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:36 smithi029 ceph-mon[30044]: pgmap v150: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 776 op/s 2024-03-29T04:35:36.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:36 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3468021338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:36.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:36 smithi029 ceph-mon[41899]: pgmap v150: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 776 op/s 2024-03-29T04:35:36.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:36 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3468021338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:38.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:38 smithi099 ceph-mon[33865]: pgmap v151: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 1.06k op/s 2024-03-29T04:35:38.859 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:38 smithi099 systemd[1]: Stopping Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:35:38.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:38 smithi029 ceph-mon[30044]: pgmap v151: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 1.06k op/s 2024-03-29T04:35:38.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:38 smithi029 ceph-mon[41899]: pgmap v151: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 1.06k op/s 2024-03-29T04:35:39.358 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:38 smithi099 podman[69789]: 2024-03-29 04:35:38.973700517 +0000 UTC m=+0.203539100 container died 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-29T04:35:39.822 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:39 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2974883970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:39.822 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:39 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:39.822 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:39 smithi099 podman[69789]: 2024-03-29 04:35:39.601521743 +0000 UTC m=+0.831360331 container cleanup 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326) 2024-03-29T04:35:39.822 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:39 smithi099 bash[69789]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:35:39.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:39 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2974883970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:39.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:39 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:39.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:39 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2974883970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:39.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:39 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:40.108 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:39 smithi099 podman[69806]: 2024-03-29 04:35:39.836732497 +0000 UTC m=+0.864394706 container remove 65870fc341f2fd6535291ce29e6b30a1c88bd3f019a97aa6a2840c27f870c83b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17) 2024-03-29T04:35:40.108 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:39 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:35:40.499 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:40 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-29T04:35:40.499 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:40 smithi099 systemd[1]: Stopped Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:35:40.499 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:40 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service: Consumed 18.366s CPU time. 2024-03-29T04:35:40.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:40 smithi099 ceph-mon[33865]: pgmap v152: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-03-29T04:35:40.813 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:40 smithi099 systemd[1]: Starting Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:35:40.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:40 smithi029 ceph-mon[30044]: pgmap v152: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-03-29T04:35:40.928 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:40 smithi029 ceph-mon[41899]: pgmap v152: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-03-29T04:35:41.108 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:40 smithi099 podman[69931]: 2024-03-29 04:35:40.730869988 +0000 UTC m=+0.038887750 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:35:41.108 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:40 smithi099 podman[69931]: 2024-03-29 04:35:40.871792768 +0000 UTC m=+0.179810504 container create 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True) 2024-03-29T04:35:41.608 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:41 smithi099 podman[69931]: 2024-03-29 04:35:41.191715492 +0000 UTC m=+0.499733238 container init 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, ceph=True) 2024-03-29T04:35:41.608 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:41 smithi099 podman[69931]: 2024-03-29 04:35:41.203475111 +0000 UTC m=+0.511492842 container start 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-03-29T04:35:41.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:41 smithi099 ceph-mgr[69975]: -- 172.21.15.99:0/4223531176 <== mon.1 v2:172.21.15.99:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x558888b5a4e0 con 0x558888b30c00 2024-03-29T04:35:41.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:41 smithi099 ceph-mgr[69975]: -- 172.21.15.99:0/4223531176 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558888b5c5a0 con 0x558888b30c00 2024-03-29T04:35:41.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:41 smithi099 bash[69931]: 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 2024-03-29T04:35:41.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:41 smithi099 systemd[1]: Started Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:35:41.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3047224288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:41.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:41.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:41.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:41.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3047224288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:41.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:41.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:41.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:41 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:42.108 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:41 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3047224288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:42.108 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:41 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:42.108 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:41 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:42.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:41 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:42.456 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:42.205+0000 7efe586b7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:35:42.456 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:42 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:42.362+0000 7efe586b7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:35:42.678 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:42 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:42] "GET /metrics HTTP/1.1" 200 242885 "" "Prometheus/2.43.0" 2024-03-29T04:35:42.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:42 smithi099 ceph-mon[33865]: pgmap v153: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-03-29T04:35:42.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:42 smithi029 ceph-mon[30044]: pgmap v153: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-03-29T04:35:42.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:42 smithi029 ceph-mon[41899]: pgmap v153: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-03-29T04:35:43.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:43.453+0000 7efe586b7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:35:43.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:43.613+0000 7efe586b7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:35:44.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:43.842+0000 7efe586b7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:35:44.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:44.001+0000 7efe586b7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:35:44.583 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:44.140+0000 7efe586b7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:35:44.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-mon[33865]: pgmap v154: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-03-29T04:35:44.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/520493285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:44.859 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:44.596+0000 7efe586b7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:35:44.859 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:44.784+0000 7efe586b7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:35:45.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:44 smithi029 ceph-mon[30044]: pgmap v154: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-03-29T04:35:45.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:44 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/520493285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:45.178 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:44 smithi029 ceph-mon[41899]: pgmap v154: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 781 op/s 2024-03-29T04:35:45.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:44 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/520493285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:45.226 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:44 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:44.931+0000 7efe586b7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:35:45.226 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:45 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:45.092+0000 7efe586b7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:35:45.608 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:45 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:45.361+0000 7efe586b7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:35:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:46 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:46 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:46.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:46 smithi099 ceph-mon[33865]: pgmap v155: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-03-29T04:35:46.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:46 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:46.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:46 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:46.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:46 smithi029 ceph-mon[30044]: pgmap v155: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-03-29T04:35:46.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:46 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:46.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:46 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:46.429 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:46 smithi029 ceph-mon[41899]: pgmap v155: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-03-29T04:35:47.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:47 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1740497090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:47.332 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:46 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:46.969+0000 7efe586b7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:35:47.332 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:47 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:47.194+0000 7efe586b7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:35:47.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:47 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1740497090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:47.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:47 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1740497090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:48.103 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:47 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:47.695+0000 7efe586b7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:35:48.103 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:47 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:47.858+0000 7efe586b7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:35:48.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:48 smithi099 ceph-mon[33865]: pgmap v156: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-03-29T04:35:48.359 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:48 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:48.198+0000 7efe586b7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:35:48.428 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:48 smithi029 ceph-mon[30044]: pgmap v156: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-03-29T04:35:48.428 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:48 smithi029 ceph-mon[41899]: pgmap v156: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-03-29T04:35:48.861 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:48 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:48.524+0000 7efe586b7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:35:49.358 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:49 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:49.041+0000 7efe586b7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:35:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:49 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:49 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:49 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3173953949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:49.859 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:49 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:49.448+0000 7efe586b7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:35:49.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:49 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:49.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:49 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:49.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:49 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3173953949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:49.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:49 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:49.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:49 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:49.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:49 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3173953949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:50.312 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:49 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:49.879+0000 7efe586b7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:35:50.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:50 smithi029 ceph-mon[30044]: pgmap v157: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-03-29T04:35:50.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:50.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:50 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:50.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:50 smithi029 ceph-mon[41899]: pgmap v157: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-03-29T04:35:50.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:50.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:50 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:51.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:50 smithi099 ceph-mon[33865]: pgmap v157: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-03-29T04:35:51.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:51.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:50 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:51.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:51 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:51.271+0000 7efe586b7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:35:52.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:52.181+0000 7efe586b7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:35:52.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:52.352+0000 7efe586b7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:35:52.611 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:52 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:52] "GET /metrics HTTP/1.1" 200 242885 "" "Prometheus/2.43.0" 2024-03-29T04:35:52.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:52 smithi029 ceph-mon[30044]: pgmap v158: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-03-29T04:35:52.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:52 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1900269815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:52.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:52 smithi029 ceph-mon[41899]: pgmap v158: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-03-29T04:35:52.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:52 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1900269815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-mon[33865]: pgmap v158: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1900269815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:52.721+0000 7efe586b7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:52] ENGINE Bus STARTING 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: CherryPy Checker: 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: The Application mounted at '' has an empty config. 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:52] ENGINE Serving on http://:::9283 2024-03-29T04:35:53.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:52] ENGINE Bus STARTED 2024-03-29T04:35:53.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:53 smithi099 ceph-mon[33865]: Standby manager daemon x restarted 2024-03-29T04:35:53.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:53 smithi099 ceph-mon[33865]: Standby manager daemon x started 2024-03-29T04:35:53.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:53 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:35:53.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:53 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:35:53.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:53 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:35:53.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:53 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:35:53.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[30044]: Standby manager daemon x restarted 2024-03-29T04:35:53.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[30044]: Standby manager daemon x started 2024-03-29T04:35:53.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:35:53.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:35:53.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:35:53.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:35:53.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[41899]: Standby manager daemon x restarted 2024-03-29T04:35:53.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[41899]: Standby manager daemon x started 2024-03-29T04:35:53.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:35:53.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:35:53.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:35:53.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:53 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:35:54.538 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:35:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: pgmap v159: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: mgrmap e22: y(active, since 4m), standbys: x 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4221123560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:54.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: pgmap v159: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: mgrmap e22: y(active, since 4m), standbys: x 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4221123560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:54.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:54.828+0000 7fd7bb58f700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setuser ceph since I am not root 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: pgmap v159: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: mgrmap e22: y(active, since 4m), standbys: x 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4221123560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:54.890 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:55.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:54] ENGINE Bus STOPPING 2024-03-29T04:35:55.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:35:55.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:54] ENGINE Bus STOPPED 2024-03-29T04:35:55.109 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:54 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:54] ENGINE Bus STARTING 2024-03-29T04:35:55.178 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ignoring --setgroup ceph since I am not root 2024-03-29T04:35:55.179 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mgr[30326]: -- 172.21.15.29:0/1033112124 <== mon.0 v2:172.21.15.29:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55f510d44c60 con 0x55f510d06800 2024-03-29T04:35:55.179 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:54 smithi029 ceph-mgr[30326]: -- 172.21.15.29:0/1033112124 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f510d309c0 con 0x55f510d06800 2024-03-29T04:35:55.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.199+0000 7efdba355700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.199+0000 7efdba355700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.199+0000 7efdba355700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.199+0000 7efdba355700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.199+0000 7efdba355700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.204+0000 7efdbf35f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.204+0000 7efdbf35f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.204+0000 7efdbf35f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.204+0000 7efdbf35f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:35:55.204+0000 7efdbf35f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:55] ENGINE Serving on http://:::9283 2024-03-29T04:35:55.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:35:55] ENGINE Bus STARTED 2024-03-29T04:35:55.928 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:55.645+0000 7f8ffdf5f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:35:55.929 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:55.782+0000 7f8ffdf5f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:35:55.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:55.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:55.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: mgrmap e23: x(active, starting, since 0.0158499s) 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:35:55.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: Manager daemon x is now available 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:55.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: mgrmap e23: x(active, starting, since 0.0158499s) 2024-03-29T04:35:55.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:35:55.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: Manager daemon x is now available 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:55.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:55 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:56.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:56.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' 2024-03-29T04:35:56.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:56.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24341 172.21.15.29:0/4253388029' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24341 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: mgrmap e23: x(active, starting, since 0.0158499s) 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:35:56.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: Manager daemon x is now available 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:56.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:55 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:57.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[30044]: Deploying cephadm binary to smithi029 2024-03-29T04:35:57.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[30044]: Deploying cephadm binary to smithi099 2024-03-29T04:35:57.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[30044]: mgrmap e24: x(active, since 1.01763s) 2024-03-29T04:35:57.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[30044]: pgmap v3: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:35:57.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:35:57.064 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:56.763+0000 7f8ffdf5f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:56.926+0000 7f8ffdf5f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[41899]: Deploying cephadm binary to smithi029 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[41899]: Deploying cephadm binary to smithi099 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[41899]: mgrmap e24: x(active, since 1.01763s) 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[41899]: pgmap v3: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:35:57.065 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:56 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:35:57.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:56 smithi099 ceph-mon[33865]: Deploying cephadm binary to smithi029 2024-03-29T04:35:57.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:56 smithi099 ceph-mon[33865]: Deploying cephadm binary to smithi099 2024-03-29T04:35:57.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:56 smithi099 ceph-mon[33865]: mgrmap e24: x(active, since 1.01763s) 2024-03-29T04:35:57.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:56 smithi099 ceph-mon[33865]: pgmap v3: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:35:57.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:56 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:35:57.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:56 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:35:57.409 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:57.077+0000 7f8ffdf5f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:35:57.409 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:57.313+0000 7f8ffdf5f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:35:57.664 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:57.422+0000 7f8ffdf5f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:35:57.928 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-mon[30044]: pgmap v4: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:35:57.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-mon[30044]: mgrmap e25: x(active, since 2s) 2024-03-29T04:35:57.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1329818657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:57.929 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:57.677+0000 7f8ffdf5f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:35:57.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-mon[41899]: pgmap v4: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:35:57.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-mon[41899]: mgrmap e25: x(active, since 2s) 2024-03-29T04:35:57.929 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1329818657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:58.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:57 smithi099 ceph-mon[33865]: pgmap v4: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:35:58.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:57 smithi099 ceph-mon[33865]: mgrmap e25: x(active, since 2s) 2024-03-29T04:35:58.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:57 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1329818657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:35:58.204 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:57 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:57.943+0000 7f8ffdf5f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:35:58.204 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:58 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:58.072+0000 7f8ffdf5f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:35:58.204 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:58 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:58.217+0000 7f8ffdf5f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:35:58.678 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:35:58 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:35:58.342+0000 7f8ffdf5f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:36:00.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:59 smithi099 ceph-mon[33865]: pgmap v5: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:36:00.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:35:59 smithi099 ceph-mon[33865]: mgrmap e26: x(active, since 4s) 2024-03-29T04:36:00.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:59 smithi029 ceph-mon[30044]: pgmap v5: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:36:00.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:35:59 smithi029 ceph-mon[30044]: mgrmap e26: x(active, since 4s) 2024-03-29T04:36:00.179 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:00.023+0000 7f8ffdf5f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:36:00.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:59 smithi029 ceph-mon[41899]: pgmap v5: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:36:00.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:35:59 smithi029 ceph-mon[41899]: mgrmap e26: x(active, since 4s) 2024-03-29T04:36:00.535 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:00.239+0000 7f8ffdf5f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:36:00.843 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:00.548+0000 7f8ffdf5f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:36:01.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:00 smithi099 ceph-mon[33865]: [29/Mar/2024:04:35:59] ENGINE Bus STARTING 2024-03-29T04:36:01.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:00 smithi099 ceph-mon[33865]: [29/Mar/2024:04:35:59] ENGINE Serving on https://172.21.15.99:7150 2024-03-29T04:36:01.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:00 smithi099 ceph-mon[33865]: [29/Mar/2024:04:35:59] ENGINE Serving on http://172.21.15.99:8765 2024-03-29T04:36:01.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:00 smithi099 ceph-mon[33865]: [29/Mar/2024:04:35:59] ENGINE Bus STARTED 2024-03-29T04:36:01.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:00 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1233444028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:01.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[30044]: [29/Mar/2024:04:35:59] ENGINE Bus STARTING 2024-03-29T04:36:01.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[30044]: [29/Mar/2024:04:35:59] ENGINE Serving on https://172.21.15.99:7150 2024-03-29T04:36:01.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[30044]: [29/Mar/2024:04:35:59] ENGINE Serving on http://172.21.15.99:8765 2024-03-29T04:36:01.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[30044]: [29/Mar/2024:04:35:59] ENGINE Bus STARTED 2024-03-29T04:36:01.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1233444028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:01.145 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:00.879+0000 7f8ffdf5f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:36:01.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[41899]: [29/Mar/2024:04:35:59] ENGINE Bus STARTING 2024-03-29T04:36:01.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[41899]: [29/Mar/2024:04:35:59] ENGINE Serving on https://172.21.15.99:7150 2024-03-29T04:36:01.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[41899]: [29/Mar/2024:04:35:59] ENGINE Serving on http://172.21.15.99:8765 2024-03-29T04:36:01.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[41899]: [29/Mar/2024:04:35:59] ENGINE Bus STARTED 2024-03-29T04:36:01.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:00 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1233444028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:01.429 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:01 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:01.157+0000 7f8ffdf5f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:36:01.851 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:01 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:01.454+0000 7f8ffdf5f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:36:02.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:01 smithi099 ceph-mon[33865]: pgmap v6: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:36:02.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:01 smithi029 ceph-mon[30044]: pgmap v6: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:36:02.179 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:01 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:01.866+0000 7f8ffdf5f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:36:02.179 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:02.004+0000 7f8ffdf5f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:36:02.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:01 smithi029 ceph-mon[41899]: pgmap v6: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-03-29T04:36:02.592 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:36:02.325Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:36:02.857 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:02 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:02.605+0000 7f8ffdf5f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:36:03.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:02 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2175880602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:03.178 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:02 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2175880602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:03.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:02 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2175880602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:04.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:03 smithi099 ceph-mon[33865]: pgmap v7: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 564 op/s 2024-03-29T04:36:04.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:03 smithi029 ceph-mon[30044]: pgmap v7: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 564 op/s 2024-03-29T04:36:04.177 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:03 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:03.719+0000 7f8ffdf5f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:36:04.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:03 smithi029 ceph-mon[41899]: pgmap v7: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 564 op/s 2024-03-29T04:36:04.429 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:04.190+0000 7f8ffdf5f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:36:04.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:36:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:04] "GET /metrics HTTP/1.1" 200 33258 "" "Prometheus/2.43.0" 2024-03-29T04:36:04.929 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:04.640+0000 7f8ffdf5f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:36:05.319 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:04.973+0000 7f8ffdf5f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:36:05.319 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:36:04] ENGINE Bus STARTING 2024-03-29T04:36:05.319 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: CherryPy Checker: 2024-03-29T04:36:05.320 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: The Application mounted at '' has an empty config. 2024-03-29T04:36:05.320 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: 2024-03-29T04:36:05.320 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:36:05] ENGINE Serving on http://:::9283 2024-03-29T04:36:05.320 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: [29/Mar/2024:04:36:05] ENGINE Bus STARTED 2024-03-29T04:36:05.679 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:36:05.332Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-03-29T04:36:06.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: pgmap v8: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 438 op/s 2024-03-29T04:36:06.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: Standby manager daemon y started 2024-03-29T04:36:06.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:36:06.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:36:06.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3868403070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: pgmap v8: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 438 op/s 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: Standby manager daemon y started 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:36:06.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3868403070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:06.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: pgmap v8: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 438 op/s 2024-03-29T04:36:06.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: Standby manager daemon y started 2024-03-29T04:36:06.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:36:06.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:36:06.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:36:06.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3785072860' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:36:06.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3868403070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:07.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:06 smithi029 ceph-mon[30044]: mgrmap e27: x(active, since 11s), standbys: y 2024-03-29T04:36:07.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:06 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:36:07.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:06 smithi029 ceph-mon[41899]: mgrmap e27: x(active, since 11s), standbys: y 2024-03-29T04:36:07.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:06 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:36:07.290 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:06 smithi099 ceph-mon[33865]: mgrmap e27: x(active, since 11s), standbys: y 2024-03-29T04:36:07.290 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:06 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:36:08.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:07 smithi029 ceph-mon[30044]: pgmap v9: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 698 op/s 2024-03-29T04:36:08.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:07 smithi029 ceph-mon[41899]: pgmap v9: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 698 op/s 2024-03-29T04:36:08.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:07 smithi099 ceph-mon[33865]: pgmap v9: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 698 op/s 2024-03-29T04:36:09.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:08 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2336942838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:09.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:08 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2336942838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:09.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:08 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2336942838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:10.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:10 smithi099 ceph-mon[33865]: pgmap v10: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 638 op/s 2024-03-29T04:36:10.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:10 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:10.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:10 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:10.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:10 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:10.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[30044]: pgmap v10: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 638 op/s 2024-03-29T04:36:10.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:10.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:10.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:10.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[41899]: pgmap v10: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 638 op/s 2024-03-29T04:36:10.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:10.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:10.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:10 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:12.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: pgmap v11: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 638 op/s 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi029 to 2481M 2024-03-29T04:36:12.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2979956199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:12.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:11 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: pgmap v11: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 638 op/s 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi029 to 2481M 2024-03-29T04:36:12.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2979956199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: pgmap v11: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 638 op/s 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi029 to 2481M 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2979956199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:11 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:12.679 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:12 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:36:14.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:13 smithi099 ceph-mon[33865]: pgmap v12: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 986 op/s 2024-03-29T04:36:14.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:13 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:14.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:13 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:14.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:13 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2386854640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:14.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[30044]: pgmap v12: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 986 op/s 2024-03-29T04:36:14.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:14.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:14.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2386854640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:14.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[41899]: pgmap v12: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 986 op/s 2024-03-29T04:36:14.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:14.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:14.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:13 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2386854640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:14.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:36:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:14] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-03-29T04:36:15.972 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:15 smithi099 ceph-mon[33865]: pgmap v13: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-03-29T04:36:16.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:15 smithi029 ceph-mon[30044]: pgmap v13: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-03-29T04:36:16.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:15 smithi029 ceph-mon[41899]: pgmap v13: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-03-29T04:36:17.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:16 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2319062922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:17.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:16 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2319062922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:17.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:16 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2319062922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:18.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:17 smithi099 ceph-mon[33865]: pgmap v14: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2024-03-29T04:36:18.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:17 smithi029 ceph-mon[30044]: pgmap v14: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2024-03-29T04:36:18.179 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:17 smithi029 ceph-mon[41899]: pgmap v14: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: pgmap v15: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: Adjusting osd_memory_target on smithi099 to 2480M 2024-03-29T04:36:20.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:20.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:20.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:36:20.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:36:20.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:19 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3140128496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:20.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: pgmap v15: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: Adjusting osd_memory_target on smithi099 to 2480M 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3140128496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: pgmap v15: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: Adjusting osd_memory_target on smithi099 to 2480M 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:36:20.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:19 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3140128496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:21.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:20 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:36:21.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:20 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:36:21.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:20 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:21.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:20 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:21.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:20 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:21.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:20 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:21.429 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:21.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:20 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:36:22.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: pgmap v16: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3182161960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:36:22.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: pgmap v16: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:36:22.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3182161960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: pgmap v16: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:36:22.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:22 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3182161960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:22.429 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:36:23.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:23 smithi029 ceph-mon[30044]: Reconfiguring daemon alertmanager.a on smithi029 2024-03-29T04:36:23.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:23 smithi029 ceph-mon[41899]: Reconfiguring daemon alertmanager.a on smithi029 2024-03-29T04:36:23.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:23 smithi099 ceph-mon[33865]: Reconfiguring daemon alertmanager.a on smithi029 2024-03-29T04:36:24.310 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:24 smithi099 ceph-mon[33865]: pgmap v17: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:36:24.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:24 smithi029 ceph-mon[30044]: pgmap v17: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:36:24.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:24 smithi029 ceph-mon[41899]: pgmap v17: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:36:24.609 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:36:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:24] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-03-29T04:36:25.179 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:25 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2671640064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:25.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:25 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:25.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:25 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2671640064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:25.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:25 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:25.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:25 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2671640064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:25.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:25 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:26.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:26 smithi099 ceph-mon[33865]: pgmap v18: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-03-29T04:36:26.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:26 smithi029 ceph-mon[30044]: pgmap v18: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-03-29T04:36:26.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:26 smithi029 ceph-mon[41899]: pgmap v18: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-03-29T04:36:26.430 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:26 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:26.930 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[78928]: ts=2024-03-29T04:36:26.559Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:36:26.930 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:26 smithi029 podman[91415]: 2024-03-29 04:36:26.610667862 +0000 UTC m=+0.198528484 container died f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:27.416 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 podman[91415]: 2024-03-29 04:36:27.164503175 +0000 UTC m=+0.752363805 container cleanup f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:27.416 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 bash[91415]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:36:27.680 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 podman[91432]: 2024-03-29 04:36:27.428769014 +0000 UTC m=+0.818061279 container remove f5ce6956110ea7ebee7f9b7bea82bc44894f00db6e4d99d11db1f7af10cc82f1 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:27.680 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 podman[91432]: 2024-03-29 04:36:27.532636654 +0000 UTC m=+0.921928929 volume remove ac59ca32c09512f7a0c8567cd603c9ce4963dd23547a540c31f997a9ca96d504 2024-03-29T04:36:28.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-mon[30044]: pgmap v19: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:36:28.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3272176791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:28.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-mon[41899]: pgmap v19: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:36:28.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3272176791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:28.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:36:28.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:28.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@alertmanager.a.service: Consumed 1.447s CPU time. 2024-03-29T04:36:28.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:27 smithi029 systemd[1]: Starting Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:28.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:28 smithi099 ceph-mon[33865]: pgmap v19: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:36:28.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:28 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3272176791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:28.608 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 podman[91544]: 2024-03-29 04:36:28.140439616 +0000 UTC m=+0.033383223 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:36:28.608 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 podman[91544]: 2024-03-29 04:36:28.266658589 +0000 UTC m=+0.159602188 volume create 71fa853b2d8b3145772fd3817d2232a969c4a29fa370f7dcdf252f3bc2b8816a 2024-03-29T04:36:28.608 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 podman[91544]: 2024-03-29 04:36:28.376518941 +0000 UTC m=+0.269462534 container create 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:29.180 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 podman[91544]: 2024-03-29 04:36:28.75199622 +0000 UTC m=+0.644939828 container init 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:29.180 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 podman[91544]: 2024-03-29 04:36:28.760484134 +0000 UTC m=+0.653427726 container start 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:29.180 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.782Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:36:29.180 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.782Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.783Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.29 port=9094 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.784Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.825Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.826Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.830Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:28.830Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 bash[91544]: 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 2024-03-29T04:36:29.181 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:28 smithi029 systemd[1]: Started Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:30.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[30044]: pgmap v20: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-03-29T04:36:30.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:30.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:30.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[30044]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:36:30.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[30044]: Reconfiguring daemon node-exporter.a on smithi029 2024-03-29T04:36:30.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[41899]: pgmap v20: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-03-29T04:36:30.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:30.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:30.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[41899]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:36:30.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:29 smithi029 ceph-mon[41899]: Reconfiguring daemon node-exporter.a on smithi029 2024-03-29T04:36:30.181 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:29 smithi029 systemd[1]: Stopping Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:30.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:29 smithi099 ceph-mon[33865]: pgmap v20: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-03-29T04:36:30.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:30.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:30.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:29 smithi099 ceph-mon[33865]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:36:30.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:29 smithi099 ceph-mon[33865]: Reconfiguring daemon node-exporter.a on smithi029 2024-03-29T04:36:31.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:30 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/889739678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:31.086 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:30 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/889739678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:31.087 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:30 smithi029 podman[91722]: 2024-03-29 04:36:30.833235583 +0000 UTC m=+0.730080716 container died 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:31.087 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:30.785Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000490573s 2024-03-29T04:36:31.261 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:30 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/889739678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:32.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:31 smithi029 ceph-mon[30044]: pgmap v21: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-03-29T04:36:32.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:31 smithi029 ceph-mon[41899]: pgmap v21: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-03-29T04:36:32.270 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:31 smithi099 ceph-mon[33865]: pgmap v21: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-03-29T04:36:32.680 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:36:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[30303]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:36:33.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:32 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1547697883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:33.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:32 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1547697883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:33.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:32 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1547697883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:34.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:33 smithi029 ceph-mon[30044]: pgmap v22: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-03-29T04:36:34.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:33 smithi029 ceph-mon[41899]: pgmap v22: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-03-29T04:36:34.310 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:33 smithi099 ceph-mon[33865]: pgmap v22: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-03-29T04:36:34.610 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:36:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:34] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-03-29T04:36:34.651 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 podman[91722]: 2024-03-29 04:36:34.335285824 +0000 UTC m=+4.232130948 container cleanup 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:34.651 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 bash[91722]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a 2024-03-29T04:36:34.904 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 podman[91742]: 2024-03-29 04:36:34.664393615 +0000 UTC m=+3.819297881 container remove 4ddfb32c84c038f550e7013db1b0d2d2bb96f158bf510e0eb817c1bb7bd0c494 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:34.904 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:36:34.904 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-29T04:36:34.904 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 systemd[1]: Stopped Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:34.904 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service: Consumed 2.712s CPU time. 2024-03-29T04:36:34.904 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:34 smithi029 systemd[1]: Starting Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:35.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 podman[91866]: 2024-03-29 04:36:35.158671109 +0000 UTC m=+0.037217846 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:36:35.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 podman[91866]: 2024-03-29 04:36:35.277137594 +0000 UTC m=+0.155684308 container create 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:35.669 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 podman[91866]: 2024-03-29 04:36:35.682202144 +0000 UTC m=+0.560748877 container init 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 podman[91866]: 2024-03-29 04:36:35.694484215 +0000 UTC m=+0.573030933 container start 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.698Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.698Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.698Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.699Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.699Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.700Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.700Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.700Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:36:35.932 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.700Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.700Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:36:35.933 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.701Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.702Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:36:35.934 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.703Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:36:35.935 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a[91879]: ts=2024-03-29T04:36:35.704Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 bash[91866]: 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 2024-03-29T04:36:35.936 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:36:35 smithi029 systemd[1]: Started Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[30044]: pgmap v23: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/455871301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[41899]: pgmap v23: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/455871301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:36.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:35 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:36.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:35 smithi099 ceph-mon[33865]: pgmap v23: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-03-29T04:36:36.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:35 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/455871301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:36.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:35 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:36.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:35 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:37.184 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:36 smithi099 ceph-mon[33865]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:36:37.184 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:36 smithi099 ceph-mon[33865]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:36:37.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:36 smithi029 ceph-mon[30044]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:36:37.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:36 smithi029 ceph-mon[30044]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:36:37.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:36 smithi029 ceph-mon[41899]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:36:37.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:36 smithi029 ceph-mon[41899]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:36:38.109 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:37 smithi099 ceph-mon[33865]: pgmap v24: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:36:38.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:37 smithi029 ceph-mon[30044]: pgmap v24: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:36:38.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:37 smithi029 ceph-mon[41899]: pgmap v24: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:36:39.180 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:38 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3114829325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:39.180 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:38 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3114829325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:39.180 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:36:38 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:36:38.789Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004747028s 2024-03-29T04:36:39.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:38 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3114829325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:40.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:39 smithi099 ceph-mon[33865]: pgmap v25: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-03-29T04:36:40.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:39 smithi029 ceph-mon[30044]: pgmap v25: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-03-29T04:36:40.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:39 smithi029 ceph-mon[41899]: pgmap v25: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-03-29T04:36:40.860 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 systemd[1]: Stopping Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:41.272 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:41.273 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2058418898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:41.273 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.963Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:36:41.273 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:36:41.273 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:36:41.273 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:36:41.273 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.964Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.965Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.965Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.965Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[66905]: ts=2024-03-29T04:36:40.965Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:36:41.274 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:40 smithi099 podman[78419]: 2024-03-29 04:36:40.976882823 +0000 UTC m=+0.154249167 container died a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:41.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:40 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:41.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:40 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2058418898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:41.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:40 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:41.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:40 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2058418898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:41.610 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:41 smithi099 podman[78419]: 2024-03-29 04:36:41.383655808 +0000 UTC m=+0.561022152 container cleanup a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:41.610 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:41 smithi099 bash[78419]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:36:42.222 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:41 smithi099 ceph-mon[33865]: pgmap v26: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-03-29T04:36:42.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:41 smithi029 ceph-mon[30044]: pgmap v26: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-03-29T04:36:42.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:41 smithi029 ceph-mon[41899]: pgmap v26: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 666 op/s 2024-03-29T04:36:42.610 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 podman[78433]: 2024-03-29 04:36:42.235266601 +0000 UTC m=+1.255034463 container remove a9fb36d03923c0524c51add147f2a4ac2764b8ff10ad3942ba6c6f331af9e3e7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:42.610 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:36:42.611 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 systemd[1]: Stopped Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:42.611 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 2.174s CPU time. 2024-03-29T04:36:42.611 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 systemd[1]: Starting Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:43.110 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 podman[78508]: 2024-03-29 04:36:42.688804194 +0000 UTC m=+0.037167148 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:36:43.110 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:42 smithi099 podman[78508]: 2024-03-29 04:36:42.828228795 +0000 UTC m=+0.176591786 container create 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:43.402 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 podman[78508]: 2024-03-29 04:36:43.131134252 +0000 UTC m=+0.479497206 container init 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 podman[78508]: 2024-03-29 04:36:43.143201306 +0000 UTC m=+0.491564257 container start 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.180Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.180Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.180Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi099 (none))" 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.181Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.181Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.185Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.186Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.189Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.189Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.190Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.190Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.436µs 2024-03-29T04:36:43.403 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.190Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.190Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.243Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.245Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.245Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.09µs wal_replay_duration=54.689202ms wbl_replay_duration=142ns total_replay_duration=54.788589ms 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.249Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.249Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.249Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.281Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=32.245856ms db_storage=1.121µs remote_storage=1.451µs web_handler=364ns query_engine=878ns scrape=766.954µs scrape_sd=149.074µs notify=15.349µs notify_sd=12.876µs rules=30.865335ms tracing=7.712µs 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.282Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:43.282Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:36:43.404 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 bash[78508]: 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 2024-03-29T04:36:43.860 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:43 smithi099 systemd[1]: Started Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:44.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-mon[33865]: pgmap v27: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:36:44.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1893089767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:44.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:44.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:43 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:44.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[30044]: pgmap v27: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:36:44.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1893089767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:44.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:44.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:44.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[41899]: pgmap v27: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:36:44.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1893089767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:44.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:44.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:43 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:36:45.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:44 smithi099 ceph-mon[33865]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:36:45.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:45 smithi099 ceph-mon[33865]: Reconfiguring daemon grafana.a on smithi099 2024-03-29T04:36:45.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:44 smithi029 ceph-mon[30044]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:36:45.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:44 smithi029 ceph-mon[30044]: Reconfiguring daemon grafana.a on smithi099 2024-03-29T04:36:45.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:44 smithi029 ceph-mon[41899]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:36:45.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:44 smithi029 ceph-mon[41899]: Reconfiguring daemon grafana.a on smithi099 2024-03-29T04:36:46.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:46 smithi099 ceph-mon[33865]: pgmap v28: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-03-29T04:36:46.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:46 smithi029 ceph-mon[30044]: pgmap v28: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-03-29T04:36:46.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:46 smithi029 ceph-mon[41899]: pgmap v28: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-03-29T04:36:47.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:47 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3949350928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:47.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:47 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3949350928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:47.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:47 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3949350928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:48.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:48 smithi099 ceph-mon[33865]: pgmap v29: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:36:48.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:48 smithi029 ceph-mon[30044]: pgmap v29: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:36:48.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:48 smithi029 ceph-mon[41899]: pgmap v29: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:36:49.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:49 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/145053234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:49.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:49 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/145053234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:49.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:49 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/145053234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:50.331 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:50 smithi029 ceph-mon[30044]: pgmap v30: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-03-29T04:36:50.331 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:50 smithi029 ceph-mon[41899]: pgmap v30: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-03-29T04:36:50.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:50 smithi099 ceph-mon[33865]: pgmap v30: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-03-29T04:36:52.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:52 smithi099 ceph-mon[33865]: pgmap v31: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-03-29T04:36:52.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:52 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2682065560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:52.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:52 smithi029 ceph-mon[30044]: pgmap v31: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-03-29T04:36:52.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:52 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2682065560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:52.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:52 smithi029 ceph-mon[41899]: pgmap v31: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2024-03-29T04:36:52.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:52 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2682065560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:54.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:54 smithi099 ceph-mon[33865]: pgmap v32: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:36:54.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:54 smithi029 ceph-mon[30044]: pgmap v32: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:36:54.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:54 smithi029 ceph-mon[41899]: pgmap v32: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:36:55.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:55 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3921153173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:55.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:55 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:55.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:55 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3921153173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:55.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:55 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:55.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:55 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3921153173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:55.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:55 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:36:56.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:56 smithi099 ceph-mon[33865]: pgmap v33: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2024-03-29T04:36:56.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:56 smithi029 ceph-mon[30044]: pgmap v33: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2024-03-29T04:36:56.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:56 smithi029 ceph-mon[41899]: pgmap v33: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2024-03-29T04:36:57.360 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:57 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1386413002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:57.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:57 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1386413002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:57.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:57 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1386413002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:36:58.110 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:36:58 smithi099 ceph-mon[33865]: pgmap v34: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:36:58.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:36:58 smithi029 ceph-mon[30044]: pgmap v34: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:36:58.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:36:58 smithi029 ceph-mon[41899]: pgmap v34: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:36:58.611 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:36:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:36:58] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.43.0" 2024-03-29T04:36:58.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:58 smithi099 systemd[1]: Stopping Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:36:58.985 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:36:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:36:58.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:36:58.985 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:58 smithi099 podman[78871]: 2024-03-29 04:36:58.684714207 +0000 UTC m=+0.170094881 container died 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, version=8.5, io.buildah.version=1.32.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, architecture=x86_64, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.openshift.tags=base rhel8, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:36:59.361 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 podman[78871]: 2024-03-29 04:36:59.094515211 +0000 UTC m=+0.579895885 container cleanup 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, release=239.1651231664, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, name=ubi8, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:36:59.361 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 bash[78871]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a 2024-03-29T04:36:59.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 podman[78885]: 2024-03-29 04:36:59.403100469 +0000 UTC m=+0.714374334 container remove 75f07d4cfd30b0b29ab06a07e735ef1ef6fe83c48e4a0b8df6dea64d450dd7bc (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, description=Ceph Grafana Container, name=ubi8, distribution-scope=public, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8) 2024-03-29T04:36:59.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:36:59.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:36:59.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:36:59.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 6.296s CPU time. 2024-03-29T04:36:59.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:00.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:00 smithi099 ceph-mon[33865]: pgmap v35: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-03-29T04:37:00.346 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:00 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3730067141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:00.346 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:36:59 smithi099 podman[78968]: 2024-03-29 04:36:59.898332721 +0000 UTC m=+0.034752238 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:37:00.347 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 podman[78968]: 2024-03-29 04:37:00.037700995 +0000 UTC m=+0.174120507 container create 122b350d62c6541939482d50ec3983873a7948d4f353ba2eb162fc90b10e347b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.component=ubi8-container, vcs-type=git, name=ubi8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64) 2024-03-29T04:37:00.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:00 smithi029 ceph-mon[30044]: pgmap v35: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-03-29T04:37:00.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:00 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3730067141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:00.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:00 smithi029 ceph-mon[41899]: pgmap v35: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-03-29T04:37:00.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:00 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3730067141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:00.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 podman[78968]: 2024-03-29 04:37:00.358280261 +0000 UTC m=+0.494699775 container init 122b350d62c6541939482d50ec3983873a7948d4f353ba2eb162fc90b10e347b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, description=Ceph Grafana Container, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:37:00.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 podman[78968]: 2024-03-29 04:37:00.369880714 +0000 UTC m=+0.506300226 container start 122b350d62c6541939482d50ec3983873a7948d4f353ba2eb162fc90b10e347b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, name=ubi8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, architecture=x86_64, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner ) 2024-03-29T04:37:00.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 bash[78968]: 122b350d62c6541939482d50ec3983873a7948d4f353ba2eb162fc90b10e347b 2024-03-29T04:37:00.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:01.019 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[78978]: logger=settings t=2024-03-29T04:37:00.663767616Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:37:01.019 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:00 smithi099 podman[79074]: 2024-03-29 04:37:00.729708757 +0000 UTC m=+0.049781054 container died 122b350d62c6541939482d50ec3983873a7948d4f353ba2eb162fc90b10e347b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , io.buildah.version=1.32.0, distribution-scope=public, name=ubi8, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:37:01.611 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:01 smithi099 systemd[1]: Stopping Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:01.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:01 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:01.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:01 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:01.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:01 smithi099 ceph-mon[33865]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:37:01.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:01 smithi099 ceph-mon[33865]: Reconfiguring daemon node-exporter.b on smithi099 2024-03-29T04:37:01.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:01 smithi099 ceph-mon[33865]: pgmap v36: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-03-29T04:37:01.612 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:01 smithi099 podman[79074]: 2024-03-29 04:37:01.346356379 +0000 UTC m=+0.666428652 container remove 122b350d62c6541939482d50ec3983873a7948d4f353ba2eb162fc90b10e347b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., distribution-scope=public, description=Ceph Grafana Container, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, release=239.1651231664) 2024-03-29T04:37:01.612 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:01 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:37:01.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:01.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:01.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[30044]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:37:01.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[30044]: Reconfiguring daemon node-exporter.b on smithi099 2024-03-29T04:37:01.931 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[30044]: pgmap v36: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-03-29T04:37:01.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:01.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:01.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[41899]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:37:01.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[41899]: Reconfiguring daemon node-exporter.b on smithi099 2024-03-29T04:37:01.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:01 smithi029 ceph-mon[41899]: pgmap v36: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-03-29T04:37:02.258 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:01 smithi099 podman[79165]: 2024-03-29 04:37:01.953246692 +0000 UTC m=+0.174099856 container died 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:37:02.258 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:01 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:37:02.258 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:01 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.088s CPU time. 2024-03-29T04:37:02.522 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:02 smithi099 podman[79165]: 2024-03-29 04:37:02.367502077 +0000 UTC m=+0.588355237 container cleanup 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:37:02.522 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:02 smithi099 bash[79165]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b 2024-03-29T04:37:02.680 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:02 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2097514896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:02.681 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:02 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2097514896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:02.792 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:02 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2097514896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:02.792 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:02 smithi099 podman[79180]: 2024-03-29 04:37:02.769145085 +0000 UTC m=+0.818320533 container remove 01b461e23b83311b1550f906bfe09fff99121eb6288465ada41b4c9f35d9bbe3 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:37:02.792 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:02 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:37:03.111 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-29T04:37:03.111 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 systemd[1]: Stopped Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:03.111 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service: Consumed 2.604s CPU time. 2024-03-29T04:37:03.111 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 systemd[1]: Starting Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:03.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-mon[33865]: pgmap v37: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:37:03.611 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 podman[79276]: 2024-03-29 04:37:03.271320063 +0000 UTC m=+0.037848038 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:37:03.930 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:03 smithi029 ceph-mon[30044]: pgmap v37: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:37:03.931 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:03 smithi029 ceph-mon[41899]: pgmap v37: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:37:03.953 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 podman[79276]: 2024-03-29 04:37:03.653730539 +0000 UTC m=+0.420258503 container create 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:37:03.953 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 podman[79276]: 2024-03-29 04:37:03.965319509 +0000 UTC m=+0.731847489 container init 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:37:04.362 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 podman[79276]: 2024-03-29 04:37:03.975092032 +0000 UTC m=+0.741620003 container start 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:37:04.362 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.979Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:37:04.362 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.980Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.983Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.983Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.983Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.983Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:37:04.363 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.984Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:37:04.364 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:37:04.365 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.985Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.986Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b[79287]: ts=2024-03-29T04:37:03.986Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:04 smithi099 bash[79276]: 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 2024-03-29T04:37:04.366 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:37:04 smithi099 systemd[1]: Started Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:05.388 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:37:05.389 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:05.390 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:37:05.390 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:37:05.390 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1706723999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:05.390 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:05.390 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:37:05.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:05.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:37:05.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:37:05.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1706723999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:05.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:37:05.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:37:05.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:05.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:37:05.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:37:05.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1706723999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:06.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:06 smithi029 ceph-mon[30044]: pgmap v38: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-03-29T04:37:06.431 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:06 smithi029 ceph-mon[41899]: pgmap v38: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-03-29T04:37:06.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:06 smithi099 ceph-mon[33865]: pgmap v38: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: Upgrade: Updating mgr.y 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: Deploying daemon mgr.y on smithi029 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: pgmap v39: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2062413539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: Upgrade: Updating mgr.y 2024-03-29T04:37:07.932 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: Deploying daemon mgr.y on smithi029 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: pgmap v39: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:07.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:07 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2062413539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:08.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: Upgrade: Updating mgr.y 2024-03-29T04:37:08.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:08.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:08.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:08.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:37:08.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:08.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: Deploying daemon mgr.y on smithi029 2024-03-29T04:37:08.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: pgmap v39: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:08.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:07 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2062413539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:08.611 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:37:08] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.43.0" 2024-03-29T04:37:09.111 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:08.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:37:09.903 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:09 smithi029 systemd[1]: Stopping Ceph mgr.y for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:10.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:09 smithi029 ceph-mon[30044]: pgmap v40: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-03-29T04:37:10.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:09 smithi029 ceph-mon[41899]: pgmap v40: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-03-29T04:37:10.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:09 smithi099 ceph-mon[33865]: pgmap v40: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-03-29T04:37:11.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:10 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:11.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:10 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1470027149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:11.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:10 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:11.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:10 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1470027149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:11.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:10 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:11.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:10 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1470027149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:11.804 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:11 smithi029 podman[92578]: 2024-03-29 04:37:11.470977117 +0000 UTC m=+1.512143365 container died a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17) 2024-03-29T04:37:12.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:11 smithi029 ceph-mon[30044]: pgmap v41: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-03-29T04:37:12.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:11 smithi029 ceph-mon[41899]: pgmap v41: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-03-29T04:37:12.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 podman[92578]: 2024-03-29 04:37:12.040895781 +0000 UTC m=+2.082062017 container cleanup a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-03-29T04:37:12.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 bash[92578]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y 2024-03-29T04:37:12.361 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:11 smithi099 ceph-mon[33865]: pgmap v41: 89 pgs: 89 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-03-29T04:37:12.361 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:12 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-03-29T04:37:12.362 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:12 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:12.362 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:12 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.088s CPU time. 2024-03-29T04:37:12.362 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:12 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:12.691 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:12 smithi099 podman[79357]: 2024-03-29 04:37:12.354293308 +0000 UTC m=+0.036039323 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:37:12.794 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 podman[92597]: 2024-03-29 04:37:12.49065708 +0000 UTC m=+1.025565934 container remove a746f2a9d0b96f79c1dd182c860e521bcac638f10f9bb721e93298ad7c88e834 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:37:12.794 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:37:12.994 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:12 smithi099 podman[79357]: 2024-03-29 04:37:12.703504043 +0000 UTC m=+0.385250049 container create 7dc0f095e7c781a3c9c15a79c567730bd5d2ead7a3ce70baeb1bc9fee77c409b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, distribution-scope=public, version=8.5, name=ubi8, release=239.1651231664, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:37:13.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service: Failed with result 'exit-code'. 2024-03-29T04:37:13.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 systemd[1]: Stopped Ceph mgr.y for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:13.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:12 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service: Consumed 1min 6.637s CPU time. 2024-03-29T04:37:13.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:13 smithi099 podman[79357]: 2024-03-29 04:37:13.006465992 +0000 UTC m=+0.688211999 container init 7dc0f095e7c781a3c9c15a79c567730bd5d2ead7a3ce70baeb1bc9fee77c409b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , vendor=Red Hat, Inc., architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, description=Ceph Grafana Container, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, name=ubi8, com.redhat.component=ubi8-container, io.buildah.version=1.32.0) 2024-03-29T04:37:13.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:13 smithi099 podman[79357]: 2024-03-29 04:37:13.018620181 +0000 UTC m=+0.700366187 container start 7dc0f095e7c781a3c9c15a79c567730bd5d2ead7a3ce70baeb1bc9fee77c409b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.buildah.version=1.32.0, release=239.1651231664, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:37:13.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:13 smithi099 bash[79357]: 7dc0f095e7c781a3c9c15a79c567730bd5d2ead7a3ce70baeb1bc9fee77c409b 2024-03-29T04:37:13.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:13 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:13.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[79368]: logger=settings t=2024-03-29T04:37:13.335172457Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:37:13.611 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:13 smithi099 podman[79404]: 2024-03-29 04:37:13.378893851 +0000 UTC m=+0.027845961 container died 7dc0f095e7c781a3c9c15a79c567730bd5d2ead7a3ce70baeb1bc9fee77c409b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, vcs-type=git, maintainer=Paul Cuzner , release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, version=8.5, com.redhat.component=ubi8-container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:37:13.647 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:13 smithi029 systemd[1]: Starting Ceph mgr.y for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:13.647 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:37:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:37:13.513Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:37:13.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:13 smithi029 podman[92743]: 2024-03-29 04:37:13.563903822 +0000 UTC m=+0.042683536 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:37:13.921 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:13 smithi029 podman[92743]: 2024-03-29 04:37:13.703962885 +0000 UTC m=+0.182742625 container create 280a22bdfeadcd1cb633d24990ee4b29f51a353343d91d35892254dd54d2b0d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:37:14.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:14 smithi029 podman[92743]: 2024-03-29 04:37:14.012316287 +0000 UTC m=+0.491096002 container init 280a22bdfeadcd1cb633d24990ee4b29f51a353343d91d35892254dd54d2b0d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:37:14.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:14 smithi029 podman[92743]: 2024-03-29 04:37:14.022613451 +0000 UTC m=+0.501393155 container start 280a22bdfeadcd1cb633d24990ee4b29f51a353343d91d35892254dd54d2b0d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-29T04:37:14.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:14 smithi029 ceph-mgr[92778]: -- 172.21.15.29:0/2897850573 <== mon.2 v2:172.21.15.29:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x55be380de4e0 con 0x55be380b5000 2024-03-29T04:37:14.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:14 smithi029 bash[92743]: 280a22bdfeadcd1cb633d24990ee4b29f51a353343d91d35892254dd54d2b0d9 2024-03-29T04:37:14.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:14 smithi029 systemd[1]: Started Ceph mgr.y for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:14.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:13 smithi029 ceph-mon[41899]: pgmap v42: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:14.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:13 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2462988763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:14.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:13 smithi029 ceph-mon[30044]: pgmap v42: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:14.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:13 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2462988763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:14.193 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:13 smithi099 ceph-mon[33865]: pgmap v42: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:14.193 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:13 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2462988763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:14.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:14 smithi099 podman[79404]: 2024-03-29 04:37:14.462462724 +0000 UTC m=+1.111414821 container remove 7dc0f095e7c781a3c9c15a79c567730bd5d2ead7a3ce70baeb1bc9fee77c409b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, architecture=x86_64, release=239.1651231664, description=Ceph Grafana Container, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, version=8.5, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:37:14.861 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:14 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:37:15.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:14.925+0000 7f21b4d18200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:37:15.181 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:15.078+0000 7f21b4d18200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:37:15.233 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:14 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:37:15.233 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:14 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.122s CPU time. 2024-03-29T04:37:15.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:15.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:15 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:15.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.681 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:15.682 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:15 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:16.558 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-mon[30044]: pgmap v43: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-03-29T04:37:16.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3496825579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:16.559 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:16.259+0000 7f21b4d18200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:37:16.559 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:16.415+0000 7f21b4d18200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:37:16.559 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-mon[41899]: pgmap v43: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-03-29T04:37:16.559 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3496825579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:16.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:16 smithi099 ceph-mon[33865]: pgmap v43: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-03-29T04:37:16.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:16 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3496825579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:16.931 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:16.639+0000 7f21b4d18200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:37:16.931 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:16.805+0000 7f21b4d18200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:37:17.368 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:16 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:16.946+0000 7f21b4d18200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:37:17.657 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:17.378+0000 7f21b4d18200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:37:17.657 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:17.590+0000 7f21b4d18200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:37:17.931 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:17.744+0000 7f21b4d18200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:37:17.931 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:17.899+0000 7f21b4d18200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:37:18.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:18 smithi029 ceph-mon[30044]: pgmap v44: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:37:18.432 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:18 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:18.169+0000 7f21b4d18200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:37:18.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:18 smithi029 ceph-mon[41899]: pgmap v44: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:37:18.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:18 smithi099 ceph-mon[33865]: pgmap v44: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:37:18.612 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:37:18] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.43.0" 2024-03-29T04:37:19.111 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:37:19.406 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:19 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/28937402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:19.406 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:19 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/28937402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:19.611 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:19 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/28937402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:20.139 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:19.817+0000 7f21b4d18200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:37:20.139 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:20.023+0000 7f21b4d18200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:37:20.410 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:20 smithi029 ceph-mon[30044]: pgmap v45: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-03-29T04:37:20.411 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:20 smithi029 ceph-mon[41899]: pgmap v45: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-03-29T04:37:20.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:20 smithi099 ceph-mon[33865]: pgmap v45: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-03-29T04:37:20.682 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:20.564+0000 7f21b4d18200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:37:21.098 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:20.706+0000 7f21b4d18200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:37:21.432 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:21.106+0000 7f21b4d18200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:37:21.771 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:21.471+0000 7f21b4d18200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:37:22.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:22.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:22.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[30044]: pgmap v46: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-03-29T04:37:22.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/286152003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:22.024 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:22.024+0000 7f21b4d18200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:37:22.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:22.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:22.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[41899]: pgmap v46: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-03-29T04:37:22.025 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:21 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/286152003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:22.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:21 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:22.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:21 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:22.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:21 smithi099 ceph-mon[33865]: pgmap v46: 89 pgs: 89 active+clean; 1.9 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-03-29T04:37:22.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:21 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/286152003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:22.681 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:22.433+0000 7f21b4d18200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:37:23.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:22.872+0000 7f21b4d18200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:37:23.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.812 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[30044]: pgmap v47: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-03-29T04:37:23.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:23.813 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-mon[41899]: pgmap v47: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-03-29T04:37:23.813 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:37:23.512Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:37:23.813 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:37:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:37:23.513Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:37:24.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:23 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:24.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:23 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:24.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:23 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:24.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:23 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:24.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:23 smithi099 ceph-mon[33865]: pgmap v47: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-03-29T04:37:24.681 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:24.298+0000 7f21b4d18200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:37:25.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:24 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/27201080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:25.141 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:24 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/27201080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:25.141 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:24 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/27201080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:25.431 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:25.173+0000 7f21b4d18200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:37:25.432 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:25.331+0000 7f21b4d18200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:37:25.516 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-03-29T04:37:25.516 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:25.517 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.122s CPU time. 2024-03-29T04:37:25.517 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:25.517 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 podman[80178]: 2024-03-29 04:37:25.344053588 +0000 UTC m=+0.036247911 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:37:25.775 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 podman[80178]: 2024-03-29 04:37:25.527867015 +0000 UTC m=+0.220061321 container create edc42e5b12fe634f355df2ad3488e7e5873aa71f38dc5df1e3a756e32bd05900 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vcs-type=git, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: pgmap v48: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: Standby manager daemon y restarted 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: Standby manager daemon y started 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:37:25.844 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:25.699+0000 7f21b4d18200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:25] ENGINE Bus STARTING 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: CherryPy Checker: 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: The Application mounted at '' has an empty config. 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:25] ENGINE Serving on http://:::9283 2024-03-29T04:37:25.845 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:25] ENGINE Bus STARTED 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: pgmap v48: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: Standby manager daemon y restarted 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: Standby manager daemon y started 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:37:25.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: pgmap v48: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: Standby manager daemon y restarted 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: Standby manager daemon y started 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:37:26.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:25 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:37:26.113 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 podman[80178]: 2024-03-29 04:37:25.839618914 +0000 UTC m=+0.531813237 container init edc42e5b12fe634f355df2ad3488e7e5873aa71f38dc5df1e3a756e32bd05900 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, name=ubi8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, architecture=x86_64, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:37:26.113 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 podman[80178]: 2024-03-29 04:37:25.849877584 +0000 UTC m=+0.542071891 container start edc42e5b12fe634f355df2ad3488e7e5873aa71f38dc5df1e3a756e32bd05900 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.openshift.expose-services=, vendor=Red Hat, Inc., release=239.1651231664, vcs-type=git, version=8.5, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:37:26.113 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 bash[80178]: edc42e5b12fe634f355df2ad3488e7e5873aa71f38dc5df1e3a756e32bd05900 2024-03-29T04:37:26.113 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:25 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:26.122 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:37:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:37:25.855Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2024-03-29T04:37:26.561 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[80192]: logger=settings t=2024-03-29T04:37:26.176121773Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:37:26.561 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:26 smithi099 podman[80223]: 2024-03-29 04:37:26.242649407 +0000 UTC m=+0.053191743 container died edc42e5b12fe634f355df2ad3488e7e5873aa71f38dc5df1e3a756e32bd05900 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, description=Ceph Grafana Container, vcs-type=git, io.openshift.tags=base rhel8, version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , release=239.1651231664, io.openshift.expose-services=) 2024-03-29T04:37:26.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:26 smithi099 ceph-mon[33865]: mgrmap e28: x(active, since 90s), standbys: y 2024-03-29T04:37:27.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:26 smithi029 ceph-mon[30044]: mgrmap e28: x(active, since 90s), standbys: y 2024-03-29T04:37:27.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:26 smithi029 ceph-mon[41899]: mgrmap e28: x(active, since 90s), standbys: y 2024-03-29T04:37:27.362 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:27 smithi099 podman[80223]: 2024-03-29 04:37:27.038785457 +0000 UTC m=+0.849327770 container remove edc42e5b12fe634f355df2ad3488e7e5873aa71f38dc5df1e3a756e32bd05900 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., description=Ceph Grafana Container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.openshift.expose-services=, io.openshift.tags=base rhel8, version=8.5, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, vcs-type=git, distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:37:27.362 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:27 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:37:27.809 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:27 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:37:27.809 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:27 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.074s CPU time. 2024-03-29T04:37:28.111 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:27 smithi099 ceph-mon[33865]: mgrmap e29: x(active, since 91s), standbys: y 2024-03-29T04:37:28.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:27 smithi099 ceph-mon[33865]: pgmap v49: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 943 op/s 2024-03-29T04:37:28.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:27 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2341350568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:28.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:27 smithi029 ceph-mon[30044]: mgrmap e29: x(active, since 91s), standbys: y 2024-03-29T04:37:28.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:27 smithi029 ceph-mon[30044]: pgmap v49: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 943 op/s 2024-03-29T04:37:28.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:27 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2341350568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:28.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:27 smithi029 ceph-mon[41899]: mgrmap e29: x(active, since 91s), standbys: y 2024-03-29T04:37:28.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:27 smithi029 ceph-mon[41899]: pgmap v49: 89 pgs: 89 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 943 op/s 2024-03-29T04:37:28.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:27 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2341350568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:28.614 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:37:28] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.43.0" 2024-03-29T04:37:28.973 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:37:29.316 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:29] ENGINE Bus STOPPING 2024-03-29T04:37:29.316 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:37:29.316 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:29] ENGINE Bus STOPPED 2024-03-29T04:37:29.316 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:29] ENGINE Bus STARTING 2024-03-29T04:37:29.337 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:28.982+0000 7efe4bcaa700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-29T04:37:29.337 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ignoring --setuser ceph since I am not root 2024-03-29T04:37:29.337 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ignoring --setgroup ceph since I am not root 2024-03-29T04:37:29.682 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:29] ENGINE Serving on http://:::9283 2024-03-29T04:37:29.682 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:37:29] ENGINE Bus STARTED 2024-03-29T04:37:29.682 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f211296b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.682 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f211296b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f211296b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f211296b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f211296b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f2116973700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f2116973700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f2116973700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.413+0000 7f2116973700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:29.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:37:29.414+0000 7f2116973700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:37:30.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.24836 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:37:30.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: mgrmap e30: y(active, starting, since 0.0101198s) 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:37:30.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: Manager daemon y is now available 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:29.859+0000 7faa71e68200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:37:30.115 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:30.017+0000 7faa71e68200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:37:30.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.24836 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: mgrmap e30: y(active, starting, since 0.0101198s) 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:30.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:37:30.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: Manager daemon y is now available 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' 2024-03-29T04:37:30.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 172.21.15.99:0/4241245250' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.24836 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: mgrmap e30: y(active, starting, since 0.0101198s) 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:30.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: Manager daemon y is now available 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:37:30.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:29 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:37:31.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:30 smithi099 ceph-mon[33865]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:37:31.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:30 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/827507689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:31.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:30 smithi099 ceph-mon[33865]: mgrmap e31: y(active, since 1.0094s) 2024-03-29T04:37:31.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:30 smithi099 ceph-mon[33865]: pgmap v3: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:31.181 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[41899]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/827507689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[41899]: mgrmap e31: y(active, since 1.0094s) 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[41899]: pgmap v3: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[30044]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/827507689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[30044]: mgrmap e31: y(active, since 1.0094s) 2024-03-29T04:37:31.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:30 smithi029 ceph-mon[30044]: pgmap v3: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:31.362 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:31.032+0000 7faa71e68200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:37:31.362 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:31.196+0000 7faa71e68200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:37:31.724 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:31.428+0000 7faa71e68200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:37:31.724 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:31.534+0000 7faa71e68200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:37:31.724 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:31.625+0000 7faa71e68200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:37:32.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: [29/Mar/2024:04:37:30] ENGINE Bus STARTING 2024-03-29T04:37:32.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: [29/Mar/2024:04:37:30] ENGINE Serving on http://172.21.15.29:8765 2024-03-29T04:37:32.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: [29/Mar/2024:04:37:30] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:37:32.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: [29/Mar/2024:04:37:30] ENGINE Bus STARTED 2024-03-29T04:37:32.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:37:32.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: Cluster is now healthy 2024-03-29T04:37:32.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:31 smithi099 ceph-mon[33865]: pgmap v4: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: [29/Mar/2024:04:37:30] ENGINE Bus STARTING 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: [29/Mar/2024:04:37:30] ENGINE Serving on http://172.21.15.29:8765 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: [29/Mar/2024:04:37:30] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: [29/Mar/2024:04:37:30] ENGINE Bus STARTED 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: Cluster is now healthy 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[30044]: pgmap v4: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:32.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: [29/Mar/2024:04:37:30] ENGINE Bus STARTING 2024-03-29T04:37:32.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: [29/Mar/2024:04:37:30] ENGINE Serving on http://172.21.15.29:8765 2024-03-29T04:37:32.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: [29/Mar/2024:04:37:30] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:37:32.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: [29/Mar/2024:04:37:30] ENGINE Bus STARTED 2024-03-29T04:37:32.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:37:32.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: Cluster is now healthy 2024-03-29T04:37:32.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:31 smithi029 ceph-mon[41899]: pgmap v4: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:32.336 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:32.032+0000 7faa71e68200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:37:32.336 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:32.186+0000 7faa71e68200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:37:32.336 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:32.345+0000 7faa71e68200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:37:32.612 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:32.522+0000 7faa71e68200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:37:32.990 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:32.811+0000 7faa71e68200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:37:33.181 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:33 smithi029 ceph-mon[30044]: mgrmap e32: y(active, since 3s) 2024-03-29T04:37:33.182 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:33 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3026212700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:33.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:33 smithi029 ceph-mon[41899]: mgrmap e32: y(active, since 3s) 2024-03-29T04:37:33.182 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:33 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3026212700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:33.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:33 smithi099 ceph-mon[33865]: mgrmap e32: y(active, since 3s) 2024-03-29T04:37:33.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:33 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3026212700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:34.301 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:34 smithi099 ceph-mon[33865]: pgmap v5: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:34.302 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:34.302+0000 7faa71e68200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:37:34.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:34 smithi029 ceph-mon[30044]: pgmap v5: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:34.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:34 smithi029 ceph-mon[41899]: pgmap v5: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:34.614 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:34.526+0000 7faa71e68200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:37:35.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:35 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1968712050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:35.112 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:35.046+0000 7faa71e68200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:37:35.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:35 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/1968712050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:35.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:35 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1968712050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:35.537 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:35.205+0000 7faa71e68200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:37:35.862 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:35.547+0000 7faa71e68200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:37:36.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:36 smithi099 ceph-mon[33865]: pgmap v6: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:36.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:36 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:36.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:36 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:36.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:35 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:35.874+0000 7faa71e68200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:37:36.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:36 smithi029 ceph-mon[30044]: pgmap v6: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:36.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:36 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:36.403 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:36 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:36.403 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:36 smithi029 ceph-mon[41899]: pgmap v6: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:37:36.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:36 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:36.404 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:36 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:36.774 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:36 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:36.375+0000 7faa71e68200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:37:37.112 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:36 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:36.784+0000 7faa71e68200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:37:37.612 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:37 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:37.204+0000 7faa71e68200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:37:37.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:37 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-03-29T04:37:37.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:37 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:37.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:37 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.074s CPU time. 2024-03-29T04:37:37.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:37 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:38.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-mon[33865]: pgmap v7: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 435 op/s 2024-03-29T04:37:38.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/4226837024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:38.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.307 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:37 smithi099 podman[81088]: 2024-03-29 04:37:37.880719773 +0000 UTC m=+0.050452391 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:37:38.307 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 podman[81088]: 2024-03-29 04:37:38.013454593 +0000 UTC m=+0.183187198 container create 931a1d32ec93988187ce51139dcf18afd62f7ca595ff0c4954bee0a86e7fe273 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-type=git, distribution-scope=public, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:37:38.307 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 podman[81088]: 2024-03-29 04:37:38.317052807 +0000 UTC m=+0.486785412 container init 931a1d32ec93988187ce51139dcf18afd62f7ca595ff0c4954bee0a86e7fe273 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , architecture=x86_64, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, version=8.5, io.openshift.expose-services=) 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[30044]: pgmap v7: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 435 op/s 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/4226837024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[41899]: pgmap v7: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 435 op/s 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/4226837024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:38.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:38 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:38.574 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 podman[81088]: 2024-03-29 04:37:38.332305229 +0000 UTC m=+0.502037830 container start 931a1d32ec93988187ce51139dcf18afd62f7ca595ff0c4954bee0a86e7fe273 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.32.0, distribution-scope=public, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8) 2024-03-29T04:37:38.574 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 bash[81088]: 931a1d32ec93988187ce51139dcf18afd62f7ca595ff0c4954bee0a86e7fe273 2024-03-29T04:37:38.574 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:38.862 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:38.582+0000 7faa71e68200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:37:38.862 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[81099]: logger=settings t=2024-03-29T04:37:38.64798813Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:37:38.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:38 smithi099 podman[81135]: 2024-03-29 04:37:38.71964704 +0000 UTC m=+0.049146115 container died 931a1d32ec93988187ce51139dcf18afd62f7ca595ff0c4954bee0a86e7fe273 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.openshift.tags=base rhel8, architecture=x86_64, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, name=ubi8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, description=Ceph Grafana Container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:37:39.612 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:39.370+0000 7faa71e68200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:37:39.612 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:39.504+0000 7faa71e68200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:37:39.613 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:39 smithi099 podman[81135]: 2024-03-29 04:37:39.465338964 +0000 UTC m=+0.794838031 container remove 931a1d32ec93988187ce51139dcf18afd62f7ca595ff0c4954bee0a86e7fe273 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, name=ubi8, maintainer=Paul Cuzner , distribution-scope=public, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, io.openshift.expose-services=, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git) 2024-03-29T04:37:39.613 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:39 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:37:40.112 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:39 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:37:40.112 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:39 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.085s CPU time. 2024-03-29T04:37:40.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: pgmap v8: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 338 op/s 2024-03-29T04:37:40.112 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: Standby manager daemon x started 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:40 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:39.855+0000 7faa71e68200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:37:39] ENGINE Bus STARTING 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: CherryPy Checker: 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: The Application mounted at '' has an empty config. 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: 2024-03-29T04:37:40.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:37:39] ENGINE Serving on http://:::9283 2024-03-29T04:37:40.114 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:39 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: [29/Mar/2024:04:37:39] ENGINE Bus STARTED 2024-03-29T04:37:40.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: pgmap v8: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 338 op/s 2024-03-29T04:37:40.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:40.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:40.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: Standby manager daemon x started 2024-03-29T04:37:40.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:37:40.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:37:40.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:37:40.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: pgmap v8: 89 pgs: 89 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 338 op/s 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: Standby manager daemon x started 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:37:40.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:40 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/3009938661' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:37:41.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[30044]: mgrmap e33: y(active, since 11s), standbys: x 2024-03-29T04:37:41.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:37:41.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3830474747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:41.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[30044]: pgmap v9: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 589 op/s 2024-03-29T04:37:41.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[41899]: mgrmap e33: y(active, since 11s), standbys: x 2024-03-29T04:37:41.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:37:41.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3830474747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:41.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:41 smithi029 ceph-mon[41899]: pgmap v9: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 589 op/s 2024-03-29T04:37:41.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:41 smithi099 ceph-mon[33865]: mgrmap e33: y(active, since 11s), standbys: x 2024-03-29T04:37:41.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:41 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:37:41.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:41 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3830474747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:41.612 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:41 smithi099 ceph-mon[33865]: pgmap v9: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 589 op/s 2024-03-29T04:37:43.612 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:43.258Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-03-29T04:37:43.613 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:43.259Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-03-29T04:37:43.613 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:43.259Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-03-29T04:37:43.613 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:43.259Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.99:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.99:8765: connect: connection refused" 2024-03-29T04:37:44.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:44 smithi099 ceph-mon[33865]: pgmap v10: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 539 op/s 2024-03-29T04:37:44.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:44 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2145834361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:44.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:44 smithi029 ceph-mon[30044]: pgmap v10: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 539 op/s 2024-03-29T04:37:44.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:44 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2145834361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:44.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:44 smithi029 ceph-mon[41899]: pgmap v10: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 539 op/s 2024-03-29T04:37:44.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:44 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2145834361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:45.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:45 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:45.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:45 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:45.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:45 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:45.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:45 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:45.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:45 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:45.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:45.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:45.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:45 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:46.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:46 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:37:46.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:46 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:37:46.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:46 smithi099 ceph-mon[33865]: pgmap v11: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 539 op/s 2024-03-29T04:37:46.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:46 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3696725590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[30044]: pgmap v11: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 539 op/s 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3696725590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:37:46.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[41899]: pgmap v11: 89 pgs: 89 active+clean; 939 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 539 op/s 2024-03-29T04:37:46.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:46 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3696725590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:47.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:37:47.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:47 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:37:47.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:37:47.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:37:47.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:47.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:47 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:48.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:48 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:37:48.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:48 smithi099 ceph-mon[33865]: pgmap v12: 89 pgs: 89 active+clean; 675 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 936 op/s 2024-03-29T04:37:48.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:48 smithi099 ceph-mon[33865]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:37:48.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:48 smithi099 ceph-mon[33865]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:37:48.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:48 smithi099 ceph-mon[33865]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:37:48.311 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:37:48 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[69944]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:37:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:37:48.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[30044]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:37:48.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[30044]: pgmap v12: 89 pgs: 89 active+clean; 675 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 936 op/s 2024-03-29T04:37:48.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[30044]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:37:48.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[30044]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:37:48.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[30044]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:37:48.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:37:48.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[41899]: pgmap v12: 89 pgs: 89 active+clean; 675 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 936 op/s 2024-03-29T04:37:48.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[41899]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:37:48.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[41899]: Reconfiguring daemon prometheus.a on smithi099 2024-03-29T04:37:48.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:48 smithi029 ceph-mon[41899]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:37:49.345 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:49 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2895393030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:49.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:49 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2895393030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:49.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:49 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2895393030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:50.333 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:50 smithi029 ceph-mon[30044]: pgmap v13: 89 pgs: 89 active+clean; 675 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-03-29T04:37:50.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:50 smithi029 ceph-mon[41899]: pgmap v13: 89 pgs: 89 active+clean; 675 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-03-29T04:37:50.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:50 smithi099 ceph-mon[33865]: pgmap v13: 89 pgs: 89 active+clean; 675 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-03-29T04:37:50.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:50 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-03-29T04:37:50.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:50 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:50.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:50 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.085s CPU time. 2024-03-29T04:37:50.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:50 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:51.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:51 smithi029 ceph-mon[30044]: pgmap v14: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:51.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:51 smithi029 ceph-mon[41899]: pgmap v14: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:51.613 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:51 smithi099 ceph-mon[33865]: pgmap v14: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:37:52.235 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:52 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/27433521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:52.235 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:52 smithi099 systemd[1]: Stopping Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:52.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:52 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/27433521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:52.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:52 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/27433521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:52.571 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 podman[82305]: 2024-03-29 04:37:52.146683168 +0000 UTC m=+1.064221032 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:37:52.572 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 podman[82305]: 2024-03-29 04:37:52.270545871 +0000 UTC m=+1.188083765 container create 24bd9625c2adfbf4005acc237e999c5896997d92304e445116e4baf17fbca629 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, maintainer=Paul Cuzner , io.buildah.version=1.32.0, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:37:52.572 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 podman[82305]: 2024-03-29 04:37:52.582043747 +0000 UTC m=+1.499581601 container init 24bd9625c2adfbf4005acc237e999c5896997d92304e445116e4baf17fbca629 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, release=239.1651231664, description=Ceph Grafana Container, name=ubi8, version=8.5, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git) 2024-03-29T04:37:52.862 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 podman[82305]: 2024-03-29 04:37:52.590384956 +0000 UTC m=+1.507922800 container start 24bd9625c2adfbf4005acc237e999c5896997d92304e445116e4baf17fbca629 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, release=239.1651231664, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:37:52.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 bash[82305]: 24bd9625c2adfbf4005acc237e999c5896997d92304e445116e4baf17fbca629 2024-03-29T04:37:52.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:53.141 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-mon[33865]: pgmap v15: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-03-29T04:37:53.141 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:52 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[82350]: logger=settings t=2024-03-29T04:37:52.900670488Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:37:53.141 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:37:53.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:53 smithi029 ceph-mon[30044]: pgmap v15: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-03-29T04:37:53.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:53 smithi029 ceph-mon[41899]: pgmap v15: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-03-29T04:37:53.553 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:37:53.553 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:37:53.553 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:37:53.553 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.151Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.152Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.152Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.152Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[78520]: ts=2024-03-29T04:37:53.152Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:37:53.554 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 podman[82372]: 2024-03-29 04:37:53.163329349 +0000 UTC m=+0.725565035 container died 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:37:53.862 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 podman[82372]: 2024-03-29 04:37:53.662145606 +0000 UTC m=+1.224381303 container cleanup 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:37:53.862 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:53 smithi099 bash[82372]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:37:53.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:53 smithi099 podman[82405]: 2024-03-29 04:37:53.680918933 +0000 UTC m=+0.766711662 container died 24bd9625c2adfbf4005acc237e999c5896997d92304e445116e4baf17fbca629 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, release=239.1651231664, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.buildah.version=1.32.0, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container) 2024-03-29T04:37:54.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:54 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2373308445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:54.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:54 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2373308445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:54.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:54 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2373308445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:54.534 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:54 smithi099 podman[82405]: 2024-03-29 04:37:54.292706586 +0000 UTC m=+1.378499301 container remove 24bd9625c2adfbf4005acc237e999c5896997d92304e445116e4baf17fbca629 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , release=239.1651231664, description=Ceph Grafana Container, name=ubi8, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, architecture=x86_64, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-03-29T04:37:54.534 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:37:54.862 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:54 smithi099 podman[82418]: 2024-03-29 04:37:54.544782708 +0000 UTC m=+1.379008517 container remove 4fdcc863c5638d36a5fff5e2aaede68698418984b4e0723a006dda9db1eed5e8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:37:55.298 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:55 smithi099 ceph-mon[33865]: pgmap v16: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-03-29T04:37:55.298 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:37:55.298 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: Stopped Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:55.299 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.631s CPU time. 2024-03-29T04:37:55.299 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: Starting Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:37:55.299 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:37:55.299 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:37:54 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.074s CPU time. 2024-03-29T04:37:55.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:55 smithi029 ceph-mon[30044]: pgmap v16: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-03-29T04:37:55.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:55 smithi029 ceph-mon[41899]: pgmap v16: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-03-29T04:37:55.612 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:55 smithi099 podman[82525]: 2024-03-29 04:37:55.216513936 +0000 UTC m=+0.041277736 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:37:55.955 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:55 smithi099 podman[82525]: 2024-03-29 04:37:55.679645876 +0000 UTC m=+0.504409667 container create cfdb8be5d010c429483d833895153b3a88acb559dccd7c0bb397180ee31c59b7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:37:56.363 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:55 smithi099 podman[82525]: 2024-03-29 04:37:55.965686801 +0000 UTC m=+0.790450596 container init cfdb8be5d010c429483d833895153b3a88acb559dccd7c0bb397180ee31c59b7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:55 smithi099 podman[82525]: 2024-03-29 04:37:55.979540672 +0000 UTC m=+0.804304467 container start cfdb8be5d010c429483d833895153b3a88acb559dccd7c0bb397180ee31c59b7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.022Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.023Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.023Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi099 (none))" 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.023Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.023Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.025Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.026Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.030Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.030Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.033Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:37:56.364 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.034Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.935µs 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.034Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.034Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 bash[82525]: cfdb8be5d010c429483d833895153b3a88acb559dccd7c0bb397180ee31c59b7 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.090Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 systemd[1]: Started Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.100Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.100Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.100Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.292µs wal_replay_duration=66.044007ms wbl_replay_duration=126ns total_replay_duration=66.312579ms 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.104Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.108Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.108Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.138Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.893781ms db_storage=1.128µs remote_storage=1.889µs web_handler=401ns query_engine=990ns scrape=597.654µs scrape_sd=105.38µs notify=16.815µs notify_sd=12.661µs rules=28.701725ms tracing=9.59µs 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.138Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:37:56.365 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:37:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:37:56.138Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:37:57.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:57.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:57.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:57.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:57.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:57.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: pgmap v17: 89 pgs: 89 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3781829607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: pgmap v17: 89 pgs: 89 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-03-29T04:37:57.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:57 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3781829607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:57.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:57.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:57.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:57.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:37:57.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:57.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: pgmap v17: 89 pgs: 89 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-03-29T04:37:57.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:57 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3781829607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:58.166 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 124012 objects 2024-03-29T04:37:58.166 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :153.795 2024-03-29T04:37:58.350 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-03-29T04:37:58.350 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-03-29T04:37:58.614 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:37:59.760 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: pgmap v18: 89 pgs: 89 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: Upgrade: Updating mgr.x 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: Deploying daemon mgr.x on smithi099 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:59.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:37:59 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/186627128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: pgmap v18: 89 pgs: 89 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: Upgrade: Updating mgr.x 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: Deploying daemon mgr.x on smithi099 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:59.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/186627128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: pgmap v18: 89 pgs: 89 active+clean; 164 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: Upgrade: Updating mgr.x 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: Deploying daemon mgr.x on smithi099 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:37:59.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:37:59 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/186627128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:01.112 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:00 smithi099 systemd[1]: Stopping Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:01.453 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 podman[82924]: 2024-03-29 04:38:01.171607229 +0000 UTC m=+0.221066403 container died 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, RELEASE=squid-8da2362, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-29T04:38:01.498 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_0' removed 2024-03-29T04:38:01.779 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:01 smithi099 ceph-mon[33865]: pgmap v19: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-03-29T04:38:01.779 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:01 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/962894199' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:38:01.779 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 podman[82924]: 2024-03-29 04:38:01.579464923 +0000 UTC m=+0.628924106 container cleanup 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-29T04:38:01.779 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 bash[82924]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:38:01.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:01 smithi029 ceph-mon[30044]: pgmap v19: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-03-29T04:38:01.846 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:01 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/962894199' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:38:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:01 smithi029 ceph-mon[41899]: pgmap v19: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-03-29T04:38:01.846 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:01 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/962894199' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:38:02.080 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 podman[82943]: 2024-03-29 04:38:01.789048064 +0000 UTC m=+0.617790309 container remove 8c31e69bdbaf50c790574d5527ba53f8d3c324d9dc191f99b41712b538acca58 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:38:02.081 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:38:02.081 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-29T04:38:02.081 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 systemd[1]: Stopped Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:02.081 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:01 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service: Consumed 42.141s CPU time. 2024-03-29T04:38:02.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 systemd[1]: Starting Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:02.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 podman[83080]: 2024-03-29 04:38:02.206206634 +0000 UTC m=+0.017748966 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:38:02.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 podman[83080]: 2024-03-29 04:38:02.315313783 +0000 UTC m=+0.126856098 container create 1c6cc4c8d8c3338929b33f1155bc548ad14511fcb50025d663b581928e23c056 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS) 2024-03-29T04:38:02.682 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:02 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/962894199' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:38:02.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:02 smithi029 ceph-mon[30044]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:38:02.683 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:02 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2028337497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:02.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:02 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/962894199' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:38:02.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:02 smithi029 ceph-mon[41899]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:38:02.683 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:02 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2028337497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:02.738 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:02 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/962894199' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:38:02.739 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:02 smithi099 ceph-mon[33865]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:38:02.739 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:02 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2028337497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:02.739 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 podman[83080]: 2024-03-29 04:38:02.743535197 +0000 UTC m=+0.555077514 container init 1c6cc4c8d8c3338929b33f1155bc548ad14511fcb50025d663b581928e23c056 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-29T04:38:02.934 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-03-29T04:38:02.935 INFO:tasks.radosbench:Beginning radosbench... 2024-03-29T04:38:02.935 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-03-29T04:38:02.935 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool create unique_pool_1 16 2024-03-29T04:38:03.007 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 podman[83080]: 2024-03-29 04:38:02.749011881 +0000 UTC m=+0.560554195 container start 1c6cc4c8d8c3338929b33f1155bc548ad14511fcb50025d663b581928e23c056 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, ceph=True, org.label-schema.schema-version=1.0) 2024-03-29T04:38:03.007 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 ceph-mgr[83118]: -- 172.21.15.99:0/2222623259 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ec77de85a0 con 0x55ec77dbd000 2024-03-29T04:38:03.007 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 bash[83080]: 1c6cc4c8d8c3338929b33f1155bc548ad14511fcb50025d663b581928e23c056 2024-03-29T04:38:03.007 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:02 smithi099 systemd[1]: Started Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:03.205 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:38:03.613 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:03.351+0000 7f08bb57c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:38:03.613 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:03.491+0000 7f08bb57c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[30044]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[41899]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:04.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:03 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:04.248 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:03 smithi099 ceph-mon[33865]: pgmap v21: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:04.248 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:03 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:04.248 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:03 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:04.249 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:03 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:04.508 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:04.257+0000 7f08bb57c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:38:04.508 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:04.359+0000 7f08bb57c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:38:04.508 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:04.516+0000 7f08bb57c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:38:04.791 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:04.612+0000 7f08bb57c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:38:04.791 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:04.700+0000 7f08bb57c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:38:05.074 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2805703084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:05.074 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:04.973+0000 7f08bb57c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:38:05.074 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:05.083+0000 7f08bb57c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:38:05.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:04 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2805703084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:05.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:04 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2805703084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:05.183 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:38:04] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-03-29T04:38:05.365 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-03-29T04:38:05.365 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:05.365 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.074s CPU time. 2024-03-29T04:38:05.365 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:05.366 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:05.186+0000 7f08bb57c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:38:05.366 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:05.311+0000 7f08bb57c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:38:05.635 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:05.489+0000 7f08bb57c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:38:05.635 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 podman[83530]: 2024-03-29 04:38:05.297191228 +0000 UTC m=+0.023632167 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:38:05.635 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 podman[83530]: 2024-03-29 04:38:05.420068432 +0000 UTC m=+0.146509358 container create eb5c57fd9e9c4c8de4eed95a5c67287ec4495054d97dbbc5c6efc823e17e67ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, architecture=x86_64, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, version=8.5, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-type=git) 2024-03-29T04:38:05.636 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 podman[83530]: 2024-03-29 04:38:05.639275525 +0000 UTC m=+0.365716453 container init eb5c57fd9e9c4c8de4eed95a5c67287ec4495054d97dbbc5c6efc823e17e67ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vendor=Red Hat, Inc., release=239.1651231664, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, architecture=x86_64, distribution-scope=public, vcs-type=git, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:38:05.636 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 podman[83530]: 2024-03-29 04:38:05.645450949 +0000 UTC m=+0.371891876 container start eb5c57fd9e9c4c8de4eed95a5c67287ec4495054d97dbbc5c6efc823e17e67ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, description=Ceph Grafana Container, architecture=x86_64, vendor=Red Hat, Inc., vcs-type=git, io.openshift.tags=base rhel8, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-03-29T04:38:05.925 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 bash[83530]: eb5c57fd9e9c4c8de4eed95a5c67287ec4495054d97dbbc5c6efc823e17e67ae 2024-03-29T04:38:05.925 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:05.925 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[83565]: logger=settings t=2024-03-29T04:38:05.806089397Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:38:05.940 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_1' created 2024-03-29T04:38:06.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:05 smithi029 ceph-mon[41899]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:06.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:05 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3120333242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-03-29T04:38:06.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:05 smithi029 ceph-mon[30044]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:06.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:05 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3120333242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-03-29T04:38:06.242 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-mon[33865]: pgmap v22: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:06.243 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:05 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3120333242' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-03-29T04:38:06.243 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:05 smithi099 podman[83648]: 2024-03-29 04:38:05.961051335 +0000 UTC m=+0.147915511 container died eb5c57fd9e9c4c8de4eed95a5c67287ec4495054d97dbbc5c6efc823e17e67ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.buildah.version=1.32.0, name=ubi8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, distribution-scope=public, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:38:06.863 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:06.543+0000 7f08bb57c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:38:06.863 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:06.695+0000 7f08bb57c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:38:06.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:06 smithi099 podman[83648]: 2024-03-29 04:38:06.519861443 +0000 UTC m=+0.706725609 container remove eb5c57fd9e9c4c8de4eed95a5c67287ec4495054d97dbbc5c6efc823e17e67ae (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, architecture=x86_64, version=8.5, distribution-scope=public, release=239.1651231664, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:38:06.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:06 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:38:07.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:06 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3120333242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-03-29T04:38:07.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:06 smithi029 ceph-mon[30044]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:38:07.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:06 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/324222325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:07.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:06 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3120333242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-03-29T04:38:07.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:06 smithi029 ceph-mon[41899]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:38:07.183 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:06 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/324222325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:07.296 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-03-29T04:38:07.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:06 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:38:07.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:06 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3120333242' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-03-29T04:38:07.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:06 smithi099 ceph-mon[33865]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:38:07.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:06 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/324222325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:07.364 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:07.054+0000 7f08bb57c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:38:07.364 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:07.162+0000 7f08bb57c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:38:07.563 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:38:07.734 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:07.375+0000 7f08bb57c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:38:07.735 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:07.570+0000 7f08bb57c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:38:07.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:07 smithi029 ceph-mon[30044]: pgmap v24: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:07.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:07 smithi029 ceph-mon[30044]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:38:07.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:07 smithi029 ceph-mon[30044]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:38:07.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:07 smithi029 ceph-mon[41899]: pgmap v24: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:07.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:07 smithi029 ceph-mon[41899]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:38:07.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:07 smithi029 ceph-mon[41899]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:38:08.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:07.873+0000 7f08bb57c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:38:08.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-mon[33865]: pgmap v24: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:08.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-mon[33865]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:38:08.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:07 smithi099 ceph-mon[33865]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:38:08.393 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:08.145+0000 7f08bb57c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:38:08.393 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:08.402+0000 7f08bb57c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:38:08.653 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:38:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[30044]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/2639986550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[41899]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:09.149 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:08 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/2639986550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:09.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-mon[33865]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:38:09.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:09.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:09.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:08 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/2639986550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:09.613 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:09.277+0000 7f08bb57c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:38:10.146 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:09 smithi099 ceph-mon[33865]: pgmap v27: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:10.146 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:09 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:10.146 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:09 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:10.146 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:09 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3268227510' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:38:10.146 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:09.894+0000 7f08bb57c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:38:10.146 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:10.000+0000 7f08bb57c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:38:10.238 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-03-29T04:38:10.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[30044]: pgmap v27: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:10.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:10.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:10.239 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3268227510' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:38:10.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[41899]: pgmap v27: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:10.239 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:10.240 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:10.240 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:09 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3268227510' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:10.232+0000 7f08bb57c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: [29/Mar/2024:04:38:10] ENGINE Bus STARTING 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: CherryPy Checker: 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: The Application mounted at '' has an empty config. 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: [29/Mar/2024:04:38:10] ENGINE Serving on http://:::9283 2024-03-29T04:38:10.592 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: [29/Mar/2024:04:38:10] ENGINE Bus STARTED 2024-03-29T04:38:11.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: Standby manager daemon x restarted 2024-03-29T04:38:11.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: Standby manager daemon x started 2024-03-29T04:38:11.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:38:11.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:38:11.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:38:11.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:38:11.207 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3268227510' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:38:11.207 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:10 smithi099 ceph-mon[33865]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:38:11.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: Standby manager daemon x restarted 2024-03-29T04:38:11.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: Standby manager daemon x started 2024-03-29T04:38:11.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:38:11.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:38:11.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3268227510' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[41899]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: Standby manager daemon x restarted 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: Standby manager daemon x started 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: from='mgr.? 172.21.15.99:0/480304356' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3268227510' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:38:11.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:10 smithi029 ceph-mon[30044]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:38:12.071 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-03-29T04:38:12.074 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-03-29T04:38:12.074 INFO:teuthology.orchestra.run:waiting for 3000 2024-03-29T04:38:12.125 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-03-29T04:38:12.125 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-03-29T04:38:12.125 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_101684 2024-03-29T04:38:12.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:38:12.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-03-29T04:38:12.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:11 smithi099 ceph-mon[33865]: pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:12.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:11 smithi099 ceph-mon[33865]: mgrmap e34: y(active, since 41s), standbys: x 2024-03-29T04:38:12.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:11 smithi099 ceph-mon[33865]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:38:12.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:11 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/649623196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:12.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[30044]: pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:12.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[30044]: mgrmap e34: y(active, since 41s), standbys: x 2024-03-29T04:38:12.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[30044]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:38:12.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/649623196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:12.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[41899]: pgmap v29: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:12.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[41899]: mgrmap e34: y(active, since 41s), standbys: x 2024-03-29T04:38:12.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[41899]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:38:12.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:11 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/649623196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:13.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3538 3522 220.113 220.125 0.00656937 0.00452739 2024-03-29T04:38:14.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: pgmap v31: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: pgmap v32: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: Upgrade: Setting container_image for all mgr 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:38:14.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:38:14.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:38:14.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:13 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:38:14.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6704 6688 208.985 197.875 0.00896584 0.0047741 2024-03-29T04:38:14.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: pgmap v31: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: pgmap v32: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: Upgrade: Setting container_image for all mgr 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:38:14.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: pgmap v30: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: pgmap v31: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: pgmap v32: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:14.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: Upgrade: Setting container_image for all mgr 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:38:14.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:13 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:38:15.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:14 smithi099 ceph-mon[33865]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:38:15.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:14 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3686829896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:15.113 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:14 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:15.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-mon[30044]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:38:15.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/3686829896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:15.119 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:15.119 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:38:14] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.43.0" 2024-03-29T04:38:15.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-mon[41899]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:38:15.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3686829896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:15.120 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:14 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:15.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10147 10131 211.046 215.188 0.00648548 0.00473083 2024-03-29T04:38:16.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 13357 13341 208.436 200.625 0.00424577 0.00479041 2024-03-29T04:38:16.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[30044]: pgmap v33: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:16.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:16.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:38:16.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:38:16.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[30044]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:16.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[41899]: pgmap v33: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:16.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:16.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:38:16.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:38:16.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:15 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:16.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:15 smithi099 ceph-mon[33865]: pgmap v33: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:38:16.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:15 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' 2024-03-29T04:38:16.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:15 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:38:16.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:15 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:38:16.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:15 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:16.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:16 smithi029 ceph-mon[30044]: Upgrade: Updating mon.a 2024-03-29T04:38:16.921 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:16 smithi029 ceph-mon[30044]: Deploying daemon mon.a on smithi029 2024-03-29T04:38:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:16 smithi029 ceph-mon[41899]: Upgrade: Updating mon.a 2024-03-29T04:38:16.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:16 smithi029 ceph-mon[41899]: Deploying daemon mon.a on smithi029 2024-03-29T04:38:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:16 smithi099 ceph-mon[33865]: Upgrade: Updating mon.a 2024-03-29T04:38:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:16 smithi099 ceph-mon[33865]: Deploying daemon mon.a on smithi029 2024-03-29T04:38:17.124 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-03-29T04:38:17.124 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:17.124 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:17.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 16068 16052 200.633 169.438 0.00671425 0.00497723 2024-03-29T04:38:17.740 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 podman[86944]: 2024-03-29 04:38:17.330568087 +0000 UTC m=+0.028095815 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:38:17.740 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 podman[86944]: 2024-03-29 04:38:17.471079349 +0000 UTC m=+0.168607060 container create 879cdfa8a68c434d3ac7c7bc1cb4eba39daf65d005221a18d131d73e94061fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, version=8.5, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-type=git, name=ubi8, io.openshift.expose-services=, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:38:18.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:17 smithi099 ceph-mon[33865]: pgmap v34: 89 pgs: 89 active+clean; 613 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-03-29T04:38:18.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:17 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/648940287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:18.027 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 podman[86944]: 2024-03-29 04:38:17.749823173 +0000 UTC m=+0.447350902 container init 879cdfa8a68c434d3ac7c7bc1cb4eba39daf65d005221a18d131d73e94061fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, version=8.5, release=239.1651231664, vcs-type=git, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, architecture=x86_64, io.openshift.expose-services=) 2024-03-29T04:38:18.027 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 podman[86944]: 2024-03-29 04:38:17.758152123 +0000 UTC m=+0.455679839 container start 879cdfa8a68c434d3ac7c7bc1cb4eba39daf65d005221a18d131d73e94061fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, architecture=x86_64, name=ubi8, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.buildah.version=1.32.0, vendor=Red Hat, Inc., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:38:18.027 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 bash[86944]: 879cdfa8a68c434d3ac7c7bc1cb4eba39daf65d005221a18d131d73e94061fcb 2024-03-29T04:38:18.027 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:17 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:18.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 19072 19056 198.483 187.75 0.00747421 0.00503195 2024-03-29T04:38:18.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:17 smithi029 ceph-mon[30044]: pgmap v34: 89 pgs: 89 active+clean; 613 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-03-29T04:38:18.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:17 smithi029 ceph-mon[30044]: from='client.? 172.21.15.29:0/648940287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:18.167 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:17 smithi029 ceph-mon[41899]: pgmap v34: 89 pgs: 89 active+clean; 613 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-03-29T04:38:18.167 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:17 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/648940287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:18.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[86958]: logger=settings t=2024-03-29T04:38:18.032607434Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:38:18.363 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:18 smithi099 podman[86993]: 2024-03-29 04:38:18.07526217 +0000 UTC m=+0.030138581 container died 879cdfa8a68c434d3ac7c7bc1cb4eba39daf65d005221a18d131d73e94061fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, architecture=x86_64, io.openshift.expose-services=, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, description=Ceph Grafana Container, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., version=8.5) 2024-03-29T04:38:18.787 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:38:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:38:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:38:18.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:18 smithi029 systemd[1]: Stopping Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:19.113 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:18 smithi099 podman[86993]: 2024-03-29 04:38:18.79725772 +0000 UTC m=+0.752134105 container remove 879cdfa8a68c434d3ac7c7bc1cb4eba39daf65d005221a18d131d73e94061fcb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.tags=base rhel8) 2024-03-29T04:38:19.113 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:18 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:38:19.126 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 21620 21604 192.876 159.25 0.00458169 0.00517881 2024-03-29T04:38:19.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[30021]: 2024-03-29T04:38:19.031+0000 7f2fe4ec6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[30021]: 2024-03-29T04:38:19.031+0000 7f2fe4ec6700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:19 smithi029 podman[102142]: 2024-03-29 04:38:19.06756644 +0000 UTC m=+0.183641354 container died 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:19] ENGINE Bus STOPPING 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:19] ENGINE Bus STOPPED 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:19] ENGINE Bus STARTING 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:19] ENGINE Serving on http://:::9283 2024-03-29T04:38:19.434 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:19] ENGINE Bus STARTED 2024-03-29T04:38:19.613 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:19 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:38:19.613 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:19 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.056s CPU time. 2024-03-29T04:38:19.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:19 smithi029 podman[102142]: 2024-03-29 04:38:19.65852695 +0000 UTC m=+0.774601873 container cleanup 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:38:19.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:19 smithi029 bash[102142]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:38:20.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 24055 24039 187.789 152.188 0.00330474 0.00531871 2024-03-29T04:38:20.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:19 smithi029 podman[102160]: 2024-03-29 04:38:19.987761711 +0000 UTC m=+0.909889194 container remove 585716eda74df0750e070a24d3c16be8760e206c6b43493ea366716a87a985a8 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:38:20.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:20 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T04:38:20.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:20 smithi029 systemd[1]: Stopped Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:20.230 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:20 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service: Consumed 14.952s CPU time. 2024-03-29T04:38:20.871 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:20 smithi029 systemd[1]: Starting Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:21.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 15 26477 26462 183.748 151.438 0.00858096 0.0054373 2024-03-29T04:38:21.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:20 smithi029 podman[102313]: 2024-03-29 04:38:20.786758622 +0000 UTC m=+0.045962645 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:38:21.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:20 smithi029 podman[102313]: 2024-03-29 04:38:20.925308766 +0000 UTC m=+0.184512821 container create 4ef356385530cdfeaab681d8cf4dbf04a593410c6487f600debd63057c46fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 podman[102313]: 2024-03-29 04:38:21.232266183 +0000 UTC m=+0.491470209 container init 4ef356385530cdfeaab681d8cf4dbf04a593410c6487f600debd63057c46fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 podman[102313]: 2024-03-29 04:38:21.244994508 +0000 UTC m=+0.504198523 container start 4ef356385530cdfeaab681d8cf4dbf04a593410c6487f600debd63057c46fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 6 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: pidfile_write: ignore empty --pid-file 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: load: jerasure load: lrc 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:38:21.559 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Git sha 0 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: DB SUMMARY 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: DB Session ID: AD2S68LJRVGA4A8AV1KL 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1157 Bytes 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2164366 ; 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:38:21.560 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.env: 0x557c8cea3c60 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.info_log: 0x557c8ee05ae0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.statistics: (nil) 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.use_fsync: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:38:21.561 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.db_log_dir: 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.wal_dir: 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.write_buffer_manager: 0x557c8edd81e0 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:38:21.562 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.unordered_write: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.row_cache: None 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.wal_filter: None 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:38:21.563 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.wal_compression: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:38:21.564 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_open_files: -1 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:38:21.565 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Compression algorithms supported: 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kZSTD supported: 0 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:38:21.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.merge_operator: 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_filter: None 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557c8ee05be0) 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:38:21.567 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_type: 0 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_index_type: 0 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_shortening: 1 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: checksum: 4 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: no_block_cache: 0 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache: 0x557c8ee091f0 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_options: 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: capacity : 536870912 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_shard_bits : 4 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: strict_capacity_limit : 0 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:38:21.568 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_cache_compressed: (nil) 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: persistent_cache: (nil) 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size: 4096 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_size_deviation: 10 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_restart_interval: 16 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: index_block_restart_interval: 1 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_block_size: 4096 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: partition_filters: 0 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: use_delta_encoding: 1 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: filter_policy: bloomfilter 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: whole_key_filtering: 1 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: verify_compression: 0 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: format_version: 5 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_index_compression: 1 2024-03-29T04:38:21.569 INFO:journalctl@ceph.mon.a.smithi029.stdout: block_align: 0 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_auto_readahead_size: 262144 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout: prepopulate_block_cache: 0 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression: NoCompression 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.num_levels: 7 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:38:21.570 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:38:21.571 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:38:21.572 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:38:21.573 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:38:21.574 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.ttl: 2592000 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:38:21.575 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12067, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fdb118fa-1198-411e-8d34-5ee0f70cc4e1 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687101307076, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-03-29T04:38:21.576 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687101318554, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2009330, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12068, "largest_seqno": 13056, "table_properties": {"data_size": 2003939, "index_size": 3224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10681, "raw_average_key_size": 23, "raw_value_size": 1993915, "raw_average_value_size": 4344, "num_data_blocks": 147, "num_entries": 459, "num_filter_entries": 459, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711687101, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fdb118fa-1198-411e-8d34-5ee0f70cc4e1", "db_session_id": "AD2S68LJRVGA4A8AV1KL", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687101318674, "job": 1, "event": "recovery_finished"} 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557c8eef6000 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: DB pointer 0x557c8eee4000 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** DB Stats ** 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:38:21.577 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: L0 1/0 1.92 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 330.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: Sum 2/0 8.44 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 330.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 330.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** Compaction Stats [default] ** 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:38:21.578 INFO:journalctl@ceph.mon.a.smithi029.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 330.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Cumulative compaction: 0.00 GB write, 76.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Interval compaction: 0.00 GB write, 76.71 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache BinnedLRUCache@0x557c8ee091f0#6 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-03-29T04:38:21.579 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: starting mon.a rank 0 at public addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] at bind addrs [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???) e3 preinit fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).mds e9 new map 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).mds e9 print_map 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: e9 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: legacy client fscid: 1 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: Filesystem 'foo' (1) 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: fs_name foo 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: epoch 8 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:38:21.580 INFO:journalctl@ceph.mon.a.smithi029.stdout: created 2024-03-29T04:32:31.571680+0000 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: modified 2024-03-29T04:32:51.533520+0000 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: tableserver 0 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: root 0 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: session_timeout 60 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: session_autoclose 300 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_file_size 1099511627776 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_xattr_size 65536 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: required_client_features {} 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: last_failure 0 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: last_failure_osd_epoch 0 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: max_mds 1 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: in 0 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: up {0=14661} 2024-03-29T04:38:21.581 INFO:journalctl@ceph.mon.a.smithi029.stdout: failed 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: damaged 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: stopped 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: data_pools [4] 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: metadata_pool 3 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: inline_data disabled 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: balancer 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: bal_rank_mask -1 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: standby_count_wanted 1 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: [mds.foo.smithi029.gsovzu{0:14661} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: Standby daemons: 2024-03-29T04:38:21.582 INFO:journalctl@ceph.mon.a.smithi029.stdout: 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout: [mds.foo.smithi099.jhvkhg{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).osd e81 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 ceph-mon[102350]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 bash[102313]: 4ef356385530cdfeaab681d8cf4dbf04a593410c6487f600debd63057c46fc96 2024-03-29T04:38:21.583 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:21 smithi029 systemd[1]: Started Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:22.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 28557 28541 178.366 129.938 0.0106596 0.0055997 2024-03-29T04:38:23.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: ignoring --setuser ceph since I am not root 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: ignoring --setgroup ceph since I am not root 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mgr[83118]: -- 172.21.15.99:0/128012971 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560a0a1f25a0 con 0x560a0a1c6c00 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3665249320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: pgmap v36: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 176 MiB/s wr, 2.82k op/s 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/489302071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:23.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: mon.a calling monitor election 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: monmap epoch 3 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: last_changed 2024-03-29T04:26:59.304920+0000 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: min_mon_release 17 (quincy) 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: election_strategy: 1 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:38:23.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: mgrmap e34: y(active, since 53s), standbys: x 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: daemon grafana.a on smithi099 is in unknown state 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: from='mgr.25016 ' entity='' 2024-03-29T04:38:23.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:22 smithi099 ceph-mon[33865]: mgrmap e35: y(active, since 53s), standbys: x 2024-03-29T04:38:23.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 30500 30484 173.189 121.438 0.00419121 0.00576643 2024-03-29T04:38:23.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:38:23.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:38:23.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:38:23.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3665249320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:23.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: pgmap v36: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 176 MiB/s wr, 2.82k op/s 2024-03-29T04:38:23.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/489302071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: mon.a calling monitor election 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: monmap epoch 3 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: last_changed 2024-03-29T04:26:59.304920+0000 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: min_mon_release 17 (quincy) 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: election_strategy: 1 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:38:23.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: mgrmap e34: y(active, since 53s), standbys: x 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: daemon grafana.a on smithi099 is in unknown state 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: from='mgr.25016 ' entity='' 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[102350]: mgrmap e35: y(active, since 53s), standbys: x 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ignoring --setuser ceph since I am not root 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ignoring --setgroup ceph since I am not root 2024-03-29T04:38:23.187 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mgr[92778]: -- 172.21.15.29:0/409961696 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f717e9c5a0 con 0x55f717e70800 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: from='mgr.25016 172.21.15.29:0/3602096270' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3665249320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: pgmap v36: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 176 MiB/s wr, 2.82k op/s 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/489302071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: mon.a calling monitor election 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: monmap epoch 3 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: last_changed 2024-03-29T04:26:59.304920+0000 2024-03-29T04:38:23.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: min_mon_release 17 (quincy) 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: election_strategy: 1 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: mgrmap e34: y(active, since 53s), standbys: x 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: daemon grafana.a on smithi099 is in unknown state 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: from='mgr.25016 ' entity='' 2024-03-29T04:38:23.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:22 smithi029 ceph-mon[41899]: mgrmap e35: y(active, since 53s), standbys: x 2024-03-29T04:38:24.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:23.676+0000 7fbc65598200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:38:24.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:23.826+0000 7fbc65598200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:38:24.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 32602 32586 169.704 131.375 0.0067698 0.00588658 2024-03-29T04:38:24.183 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:23.876+0000 7f94f9862200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:38:24.183 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:24.052+0000 7f94f9862200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:38:25.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:24.852+0000 7fbc65598200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:38:25.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:24.996+0000 7fbc65598200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:38:25.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 34511 34495 165.827 119.312 0.0090631 0.00602421 2024-03-29T04:38:25.376 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 1 2024-03-29T04:38:25.475 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:25 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/616262264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:25.475 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:25.202+0000 7fbc65598200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:38:25.476 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:25.351+0000 7fbc65598200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:38:25.642 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/616262264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:25.642 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:25.258+0000 7f94f9862200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:38:25.642 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:25.433+0000 7f94f9862200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:38:25.642 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:25.650+0000 7f94f9862200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:38:25.643 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/616262264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:38:25.861 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:25.483+0000 7fbc65598200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:38:25.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:25.791+0000 7f94f9862200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:38:26.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:25.869+0000 7fbc65598200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:38:26.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:26.044+0000 7fbc65598200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:38:26.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 36282 36266 161.887 110.688 0.00526168 0.00616873 2024-03-29T04:38:26.433 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:25 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:25.944+0000 7f94f9862200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:38:26.437 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:26.168+0000 7fbc65598200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:38:26.437 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:26.271+0000 7fbc65598200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:38:26.437 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:26.446+0000 7fbc65598200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:38:26.706 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:26.303+0000 7f94f9862200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:38:26.706 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:26.416+0000 7f94f9862200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:38:26.706 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:26.537+0000 7f94f9862200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:38:27.127 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 37346 37330 155.528 66.5 0.00699967 0.00642411 2024-03-29T04:38:27.183 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:26.715+0000 7f94f9862200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:38:27.183 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:26.932+0000 7f94f9862200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:38:28.113 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:27 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:27.831+0000 7fbc65598200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:38:28.114 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:27 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:27.966+0000 7fbc65598200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:38:28.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 38931 38915 151.998 99.0625 0.0075836 0.00657391 2024-03-29T04:38:28.863 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:28.420+0000 7fbc65598200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:38:28.863 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:28.560+0000 7fbc65598200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:38:28.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:28.408+0000 7f94f9862200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:38:28.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:28.621+0000 7f94f9862200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:38:29.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 40235 40219 147.851 81.5 0.00229468 0.00668804 2024-03-29T04:38:29.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:28.875+0000 7fbc65598200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:38:29.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:29.095+0000 7fbc65598200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:38:29.622 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:29.382+0000 7fbc65598200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:38:29.624 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:29 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-03-29T04:38:29.624 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:29 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:29.624 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:29 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Consumed 1.056s CPU time. 2024-03-29T04:38:29.624 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:29 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:29.624 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:29 smithi099 podman[87165]: 2024-03-29 04:38:29.507833465 +0000 UTC m=+0.017533786 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:38:29.624 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:29 smithi099 podman[87165]: 2024-03-29 04:38:29.631249525 +0000 UTC m=+0.140949827 container create 47ef91356352c8c08fe3fca3ef2895f62178df9fdab9d50e6dead7c2ed05ae25 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=ubi8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, version=8.5, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0) 2024-03-29T04:38:29.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:29.101+0000 7f94f9862200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:38:29.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:29.192+0000 7f94f9862200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:38:29.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:29.392+0000 7f94f9862200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:38:29.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:29.624+0000 7f94f9862200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:38:30.056 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:29.702+0000 7fbc65598200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:38:30.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 41212 41196 143.029 61.0625 0.00557501 0.00698615 2024-03-29T04:38:30.350 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:30.065+0000 7fbc65598200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:38:30.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:30 smithi099 podman[87165]: 2024-03-29 04:38:30.251045004 +0000 UTC m=+0.760745308 container init 47ef91356352c8c08fe3fca3ef2895f62178df9fdab9d50e6dead7c2ed05ae25 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, name=ubi8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8.5) 2024-03-29T04:38:30.350 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:30 smithi099 podman[87165]: 2024-03-29 04:38:30.260034764 +0000 UTC m=+0.769735072 container start 47ef91356352c8c08fe3fca3ef2895f62178df9fdab9d50e6dead7c2ed05ae25 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, name=ubi8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, maintainer=Paul Cuzner , io.buildah.version=1.32.0, distribution-scope=public, vendor=Red Hat, Inc., version=8.5, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:38:30.433 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:30.109+0000 7f94f9862200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:38:30.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:30 smithi099 bash[87165]: 47ef91356352c8c08fe3fca3ef2895f62178df9fdab9d50e6dead7c2ed05ae25 2024-03-29T04:38:30.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:30 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:30.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[87183]: logger=settings t=2024-03-29T04:38:30.497242287Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:38:30.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:30 smithi099 podman[87222]: 2024-03-29 04:38:30.527505586 +0000 UTC m=+0.022260751 container died 47ef91356352c8c08fe3fca3ef2895f62178df9fdab9d50e6dead7c2ed05ae25 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.openshift.expose-services=) 2024-03-29T04:38:30.933 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:30.494+0000 7f94f9862200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:38:30.934 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:30.837+0000 7f94f9862200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:38:31.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 42715 42699 140.445 93.9375 0.00586467 0.00711443 2024-03-29T04:38:31.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:31.300+0000 7fbc65598200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:38:31.364 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:31 smithi099 podman[87222]: 2024-03-29 04:38:31.115046244 +0000 UTC m=+0.609801418 container remove 47ef91356352c8c08fe3fca3ef2895f62178df9fdab9d50e6dead7c2ed05ae25 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, vendor=Red Hat, Inc., description=Ceph Grafana Container, release=239.1651231664, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:38:31.364 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:31 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:38:31.863 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:31 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:38:32.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:38:32.138181+0000 min lat: 0.00132983 max lat: 0.583804 avg lat: 0.00727846 2024-03-29T04:38:32.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:38:32.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 43953 43937 137.291 77.375 0.00605298 0.00727846 2024-03-29T04:38:32.197 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:38:32.363 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:32.098+0000 7fbc65598200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:38:32.364 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:32.249+0000 7fbc65598200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:38:32.433 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:32.120+0000 7f94f9862200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:38:32.689 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:32.621+0000 7fbc65598200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:38:32.689 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: [29/Mar/2024:04:38:32] ENGINE Bus STARTING 2024-03-29T04:38:32.750 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:38:32.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[102350]: Standby manager daemon x restarted 2024-03-29T04:38:32.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[102350]: Standby manager daemon x started 2024-03-29T04:38:32.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[102350]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:38:32.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[102350]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:38:32.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[102350]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:38:32.952 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[102350]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:38:32.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[41899]: Standby manager daemon x restarted 2024-03-29T04:38:32.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[41899]: Standby manager daemon x started 2024-03-29T04:38:32.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:38:32.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:38:32.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:38:32.953 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:32 smithi029 ceph-mon[41899]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:38:33.055 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-mon[33865]: Standby manager daemon x restarted 2024-03-29T04:38:33.055 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-mon[33865]: Standby manager daemon x started 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-mon[33865]: from='mgr.? 172.21.15.99:0/852874307' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: CherryPy Checker: 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: The Application mounted at '' has an empty config. 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: [29/Mar/2024:04:38:32] ENGINE Serving on http://:::9283 2024-03-29T04:38:33.056 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:38:32 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x[83095]: [29/Mar/2024:04:38:32] ENGINE Bus STARTED 2024-03-29T04:38:33.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 45486 45470 135.315 95.8125 0.0197074 0.0073849 2024-03-29T04:38:33.329 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:33.156+0000 7f94f9862200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:38:33.683 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:33.337+0000 7f94f9862200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:38:33.961 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-mon[102350]: mgrmap e36: y(active, since 63s), standbys: x 2024-03-29T04:38:33.961 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:33.740+0000 7f94f9862200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:38:33.961 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:33] ENGINE Bus STARTING 2024-03-29T04:38:33.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-mon[41899]: mgrmap e36: y(active, since 63s), standbys: x 2024-03-29T04:38:33.961 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c[41876]: 2024-03-29T04:38:33.755+0000 7f8903c7f700 -1 mon.c@2(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:38:34.065 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:33 smithi099 ceph-mon[33865]: mgrmap e36: y(active, since 63s), standbys: x 2024-03-29T04:38:34.065 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:33 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b[33829]: 2024-03-29T04:38:33.754+0000 7f3d77c73700 -1 mon.b@1(peon).osd e82 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:38:34.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 47046 47030 133.596 97.5 0.006232 0.00747825 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: CherryPy Checker: 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: The Application mounted at '' has an empty config. 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:34] ENGINE Serving on http://:::9283 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:38:34] ENGINE Bus STARTED 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.205+0000 7f9460347700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.223 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.205+0000 7f9460347700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.205+0000 7f9460347700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.205+0000 7f9460347700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.205+0000 7f9460347700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.213+0000 7f94562b3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.213+0000 7f94562b3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.213+0000 7f94562b3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.214+0000 7f94562b3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.224 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.214+0000 7f94562b3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:38:34.792+0000 7f94b7a0c700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: Active manager daemon y restarted 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: Activating manager daemon y 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: mgrmap e37: y(active, starting, since 0.0143073s), standbys: x 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: failed to encode map e82 with expected crc 2024-03-29T04:38:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: failed to encode map e82 with expected crc 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: failed to encode map e82 with expected crc 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:34.899 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: Manager daemon y is now available 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: Active manager daemon y restarted 2024-03-29T04:38:34.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: Activating manager daemon y 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: mgrmap e37: y(active, starting, since 0.0143073s), standbys: x 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: failed to encode map e82 with expected crc 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: failed to encode map e82 with expected crc 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: failed to encode map e82 with expected crc 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:38:34.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: Manager daemon y is now available 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:34.902 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:38:34.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: Active manager daemon y restarted 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: Activating manager daemon y 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: mgrmap e37: y(active, starting, since 0.0143073s), standbys: x 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: failed to encode map e82 with expected crc 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: failed to encode map e82 with expected crc 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: failed to encode map e82 with expected crc 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:38:35.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi029.gsovzu"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.jhvkhg"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:38:35.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:38:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:38:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: Manager daemon y is now available 2024-03-29T04:38:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:38:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:38:35.128 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 48307 48291 131.214 78.8125 0.0107279 0.00761528 2024-03-29T04:38:35.183 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:38:34] "GET /metrics HTTP/1.1" 200 33539 "" "Prometheus/2.43.0" 2024-03-29T04:38:35.551 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (2m) 57s ago 7m 17.2M - 0.25.0 c8568f914cd2 6fb7bc148154 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi099 *:3000 unknown 27s ago 6m - - 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.gsovzu smithi029 running (5m) 57s ago 5m 14.6M - 17.2.7-789-g4213e17c cf22eb8683be e6b24be4eed9 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi099.jhvkhg smithi099 running (5m) 27s ago 5m 16.0M - 17.2.7-789-g4213e17c cf22eb8683be 2d2ff55bb03a 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi099 *:8443,9283,8765 running (33s) 27s ago 11m 69.0M - 19.0.0-1959-g8da23624 54112f57da44 1c6cc4c8d8c3 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (81s) 57s ago 13m 477M - 19.0.0-1959-g8da23624 54112f57da44 280a22bdfead 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (13m) 57s ago 13m 53.1M 2048M 17.2.7-789-g4213e17c cf22eb8683be 585716eda74d 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi099 running (11m) 27s ago 11m 45.8M 2048M 17.2.7-789-g4213e17c cf22eb8683be 48c38660bc12 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (11m) 57s ago 11m 44.6M 2048M 17.2.7-789-g4213e17c cf22eb8683be 0159b319c9a5 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (2m) 57s ago 6m 7721k - 1.5.0 0da6a335fe13 9026c9798e07 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi099 *:9100 running (92s) 27s ago 6m 7982k - 1.5.0 0da6a335fe13 52f151f28461 2024-03-29T04:38:35.552 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (11m) 57s ago 10m 362M 2481M 17.2.7-789-g4213e17c cf22eb8683be 5e5c90bac829 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (10m) 57s ago 10m 238M 2481M 17.2.7-789-g4213e17c cf22eb8683be 6532f8ba40e9 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (10m) 57s ago 10m 306M 2481M 17.2.7-789-g4213e17c cf22eb8683be 817a836d7edf 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (9m) 57s ago 9m 161M 2481M 17.2.7-789-g4213e17c cf22eb8683be a4cb24ddb406 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi099 running (9m) 27s ago 9m 281M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1fab66997673 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi099 running (8m) 27s ago 8m 407M 2480M 17.2.7-789-g4213e17c cf22eb8683be 78420bf6aaa9 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi099 running (8m) 27s ago 8m 213M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1f39d06fae80 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi099 running (8m) 27s ago 8m 227M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6adf0209fbb4 2024-03-29T04:38:35.553 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi099 *:9095 running (40s) 27s ago 7m 47.3M - 2.43.0 a07b618ecd1d cfdb8be5d010 2024-03-29T04:38:35.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:35 smithi029 ceph-mon[102350]: mgrmap e38: y(active, since 1.0242s), standbys: x 2024-03-29T04:38:35.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:35 smithi029 ceph-mon[41899]: mgrmap e38: y(active, since 1.0242s), standbys: x 2024-03-29T04:38:36.044 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:35 smithi099 ceph-mon[33865]: mgrmap e38: y(active, since 1.0242s), standbys: x 2024-03-29T04:38:36.045 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:35 smithi099 ceph-mon[33865]: pgmap v3: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:36.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 49420 49404 128.645 69.5625 0.00847299 0.00776824 2024-03-29T04:38:36.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:35 smithi029 ceph-mon[102350]: pgmap v3: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:36.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:35 smithi029 ceph-mon[41899]: pgmap v3: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:37.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: [29/Mar/2024:04:38:35] ENGINE Bus STARTING 2024-03-29T04:38:37.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: [29/Mar/2024:04:38:35] ENGINE Serving on http://172.21.15.29:8765 2024-03-29T04:38:37.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: from='client.34137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:37.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: [29/Mar/2024:04:38:35] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:38:37.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: [29/Mar/2024:04:38:35] ENGINE Bus STARTED 2024-03-29T04:38:37.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:38:37.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: Cluster is now healthy 2024-03-29T04:38:37.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: pgmap v4: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:37.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:36 smithi099 ceph-mon[33865]: mgrmap e39: y(active, since 2s), standbys: x 2024-03-29T04:38:37.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 50732 50716 126.778 82 0.00326167 0.00787828 2024-03-29T04:38:37.183 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: [29/Mar/2024:04:38:35] ENGINE Bus STARTING 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: [29/Mar/2024:04:38:35] ENGINE Serving on http://172.21.15.29:8765 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: from='client.34137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: [29/Mar/2024:04:38:35] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: [29/Mar/2024:04:38:35] ENGINE Bus STARTED 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: Cluster is now healthy 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: pgmap v4: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:37.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[102350]: mgrmap e39: y(active, since 2s), standbys: x 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: [29/Mar/2024:04:38:35] ENGINE Bus STARTING 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: [29/Mar/2024:04:38:35] ENGINE Serving on http://172.21.15.29:8765 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: from='client.34137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: [29/Mar/2024:04:38:35] ENGINE Serving on https://172.21.15.29:7150 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: [29/Mar/2024:04:38:35] ENGINE Bus STARTED 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: Cluster is now healthy 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: pgmap v4: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:37.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:36 smithi029 ceph-mon[41899]: mgrmap e39: y(active, since 2s), standbys: x 2024-03-29T04:38:38.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 51997 51981 124.943 79.0625 0.00695316 0.00799879 2024-03-29T04:38:38.279 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:37 smithi099 ceph-mon[33865]: pgmap v5: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:38.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:37 smithi029 ceph-mon[102350]: pgmap v5: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:38.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:38 smithi029 ceph-mon[41899]: pgmap v5: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:38.991 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:38:38 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:38:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:38:39.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 53317 53301 123.371 82.5 0.00726842 0.00809985 2024-03-29T04:38:39.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:39 smithi099 ceph-mon[33865]: mgrmap e40: y(active, since 4s), standbys: x 2024-03-29T04:38:39.433 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:39 smithi029 ceph-mon[102350]: mgrmap e40: y(active, since 4s), standbys: x 2024-03-29T04:38:39.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:39 smithi029 ceph-mon[41899]: mgrmap e40: y(active, since 4s), standbys: x 2024-03-29T04:38:40.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 55014 54998 122.752 106.062 0.0112765 0.00814081 2024-03-29T04:38:41.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:40 smithi099 ceph-mon[33865]: pgmap v6: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:41.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:40 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:41.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:40 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:41.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 56157 56141 120.982 71.4375 0.0114078 0.00825547 2024-03-29T04:38:41.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:40 smithi029 ceph-mon[102350]: pgmap v6: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:41.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:41.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:41.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:40 smithi029 ceph-mon[41899]: pgmap v6: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail 2024-03-29T04:38:41.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:40 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:41.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:40 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:41.872 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:41 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-03-29T04:38:41.872 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:41 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:41.872 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:41 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:41.872 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:41 smithi099 podman[88162]: 2024-03-29 04:38:41.756506997 +0000 UTC m=+0.016994199 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:38:42.129 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 57426 57410 119.593 79.3125 0.0027324 0.00834844 2024-03-29T04:38:42.614 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:42 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:42.614 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:42 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:42.614 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:42 smithi099 ceph-mon[33865]: pgmap v7: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 30 KiB/s rd, 59 MiB/s wr, 961 op/s 2024-03-29T04:38:42.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:41 smithi099 podman[88162]: 2024-03-29 04:38:41.880748141 +0000 UTC m=+0.141235342 container create b8e55affd508c420ec56ae83bbc550b2620d8fa46e0f29907ebe6722790f8200 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, description=Ceph Grafana Container, vendor=Red Hat, Inc., maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container) 2024-03-29T04:38:42.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:42.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:42.650 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:42 smithi029 ceph-mon[102350]: pgmap v7: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 30 KiB/s rd, 59 MiB/s wr, 961 op/s 2024-03-29T04:38:42.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:42 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:42.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:42 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:42.651 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:42 smithi029 ceph-mon[41899]: pgmap v7: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 30 KiB/s rd, 59 MiB/s wr, 961 op/s 2024-03-29T04:38:43.067 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:42 smithi099 podman[88162]: 2024-03-29 04:38:42.784138209 +0000 UTC m=+1.044625411 container init b8e55affd508c420ec56ae83bbc550b2620d8fa46e0f29907ebe6722790f8200 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, build-date=2022-04-29T11:48:11.055897) 2024-03-29T04:38:43.067 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:42 smithi099 podman[88162]: 2024-03-29 04:38:42.792260413 +0000 UTC m=+1.052747615 container start b8e55affd508c420ec56ae83bbc550b2620d8fa46e0f29907ebe6722790f8200 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, description=Ceph Grafana Container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, name=ubi8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=) 2024-03-29T04:38:43.067 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:42 smithi099 bash[88162]: b8e55affd508c420ec56ae83bbc550b2620d8fa46e0f29907ebe6722790f8200 2024-03-29T04:38:43.067 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:42 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:43.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 58680 58664 118.263 78.375 0.00521894 0.0084502 2024-03-29T04:38:43.364 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:43 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[88186]: logger=settings t=2024-03-29T04:38:43.075380167Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:38:44.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 59920 59904 116.989 77.5 0.00428592 0.00854283 2024-03-29T04:38:44.540 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:44 smithi099 podman[88245]: 2024-03-29 04:38:44.439816074 +0000 UTC m=+1.347014502 container remove b8e55affd508c420ec56ae83bbc550b2620d8fa46e0f29907ebe6722790f8200 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, vcs-type=git, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897) 2024-03-29T04:38:44.540 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:44 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:38:45.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:44 smithi099 ceph-mon[33865]: pgmap v8: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 23 KiB/s rd, 46 MiB/s wr, 747 op/s 2024-03-29T04:38:45.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 60979 60963 115.449 66.1875 0.00963164 0.00865686 2024-03-29T04:38:45.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:44 smithi029 ceph-mon[102350]: pgmap v8: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 23 KiB/s rd, 46 MiB/s wr, 747 op/s 2024-03-29T04:38:45.184 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:38:44] "GET /metrics HTTP/1.1" 200 33539 "" "Prometheus/2.43.0" 2024-03-29T04:38:45.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:44 smithi029 ceph-mon[41899]: pgmap v8: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 23 KiB/s rd, 46 MiB/s wr, 747 op/s 2024-03-29T04:38:45.352 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:38:45.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:45 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:38:46.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 15 62457 62442 114.772 92.4375 0.0134673 0.00870765 2024-03-29T04:38:46.306 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:38:47.095 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:46 smithi029 ceph-mon[102350]: pgmap v9: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 19 KiB/s rd, 73 MiB/s wr, 1.17k op/s 2024-03-29T04:38:47.095 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:46 smithi029 ceph-mon[41899]: pgmap v9: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 19 KiB/s rd, 73 MiB/s wr, 1.17k op/s 2024-03-29T04:38:47.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:46 smithi099 ceph-mon[33865]: pgmap v9: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 19 KiB/s rd, 73 MiB/s wr, 1.17k op/s 2024-03-29T04:38:47.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 63882 63866 114.036 89 0.0237007 0.00876338 2024-03-29T04:38:48.130 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 65434 65418 113.562 97 0.0100146 0.00880038 2024-03-29T04:38:48.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:48 smithi029 ceph-mon[102350]: pgmap v10: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 17 KiB/s rd, 66 MiB/s wr, 1.07k op/s 2024-03-29T04:38:48.934 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:48 smithi029 ceph-mon[41899]: pgmap v10: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 17 KiB/s rd, 66 MiB/s wr, 1.07k op/s 2024-03-29T04:38:49.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:48 smithi099 ceph-mon[33865]: pgmap v10: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 17 KiB/s rd, 66 MiB/s wr, 1.07k op/s 2024-03-29T04:38:49.114 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:38:48 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:38:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:38:49.132 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 66878 66862 112.925 90.25 0.00864378 0.00885059 2024-03-29T04:38:49.806 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:38:50.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 68166 68150 112.072 80.5 0.0164593 0.00891756 2024-03-29T04:38:50.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:50.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: pgmap v11: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 17 KiB/s rd, 80 MiB/s wr, 1.29k op/s 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: from='client.25274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: pgmap v11: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 17 KiB/s rd, 80 MiB/s wr, 1.29k op/s 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: from='client.25274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:38:50.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:49 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:38:50.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:50.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: pgmap v11: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 17 KiB/s rd, 80 MiB/s wr, 1.29k op/s 2024-03-29T04:38:50.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: from='client.25274 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:50.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:38:50.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:51.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 69162 69146 110.794 62.25 0.0036079 0.00902073 2024-03-29T04:38:51.725 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:38:52.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:38:52.142435+0000 min lat: 0.00132983 max lat: 0.583804 avg lat: 0.00909353 2024-03-29T04:38:52.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:38:52.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 70368 70352 109.908 75.375 0.00675352 0.00909353 2024-03-29T04:38:52.379 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:52 smithi029 ceph-mon[102350]: pgmap v12: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 17 KiB/s rd, 103 MiB/s wr, 1.66k op/s 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:52 smithi029 ceph-mon[41899]: pgmap v12: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 17 KiB/s rd, 103 MiB/s wr, 1.66k op/s 2024-03-29T04:38:53.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:52 smithi099 ceph-mon[33865]: pgmap v12: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 17 KiB/s rd, 103 MiB/s wr, 1.66k op/s 2024-03-29T04:38:53.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 71532 71516 109.002 72.75 0.0105521 0.00916896 2024-03-29T04:38:54.133 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 72901 72885 108.444 85.5625 0.0126867 0.00921659 2024-03-29T04:38:55.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-mon[102350]: pgmap v13: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-03-29T04:38:55.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:55.102 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:55.102 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:38:54] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-03-29T04:38:55.102 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-mon[41899]: pgmap v13: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-03-29T04:38:55.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:55.103 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:54 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:55.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:54 smithi099 ceph-mon[33865]: pgmap v13: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-03-29T04:38:55.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:55.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:55.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 74116 74100 107.686 75.9375 0.0104434 0.00928122 2024-03-29T04:38:55.647 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-03-29T04:38:55.648 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:55.648 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 systemd[1]: Starting Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:38:55.950 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 podman[91107]: 2024-03-29 04:38:55.558611855 +0000 UTC m=+0.027677095 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:38:55.950 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 podman[91107]: 2024-03-29 04:38:55.696090623 +0000 UTC m=+0.165155891 container create ccd7caa9d6fc9b146922a80b9e0512de48778bd79025bbc8bb608f5bd0f2fe3c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, version=8.5, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, vcs-type=git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:38:56.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 75327 75311 106.959 75.6875 0.0353144 0.00934356 2024-03-29T04:38:56.226 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 podman[91107]: 2024-03-29 04:38:55.959225488 +0000 UTC m=+0.428290711 container init ccd7caa9d6fc9b146922a80b9e0512de48778bd79025bbc8bb608f5bd0f2fe3c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, io.openshift.expose-services=, description=Ceph Grafana Container, architecture=x86_64, name=ubi8, distribution-scope=public, maintainer=Paul Cuzner , vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:38:56.226 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:55 smithi099 podman[91107]: 2024-03-29 04:38:55.966473602 +0000 UTC m=+0.435538806 container start ccd7caa9d6fc9b146922a80b9e0512de48778bd79025bbc8bb608f5bd0f2fe3c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, version=8.5, build-date=2022-04-29T11:48:11.055897, name=ubi8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container) 2024-03-29T04:38:56.226 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:56 smithi099 bash[91107]: ccd7caa9d6fc9b146922a80b9e0512de48778bd79025bbc8bb608f5bd0f2fe3c 2024-03-29T04:38:56.226 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:56 smithi099 systemd[1]: Started Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:38:56.226 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a[91119]: logger=settings t=2024-03-29T04:38:56.200733988Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:38:56.548 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:56 smithi099 podman[91156]: 2024-03-29 04:38:56.234442672 +0000 UTC m=+0.024293495 container died ccd7caa9d6fc9b146922a80b9e0512de48778bd79025bbc8bb608f5bd0f2fe3c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64) 2024-03-29T04:38:57.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:56 smithi099 ceph-mon[33865]: pgmap v14: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-03-29T04:38:57.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:56 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:57.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:56 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:57.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:56 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:57.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:56 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:57.118 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:56 smithi099 podman[91156]: 2024-03-29 04:38:56.91549528 +0000 UTC m=+0.705346091 container remove ccd7caa9d6fc9b146922a80b9e0512de48778bd79025bbc8bb608f5bd0f2fe3c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, distribution-scope=public, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., architecture=x86_64) 2024-03-29T04:38:57.118 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:56 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:38:57.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 76549 76533 106.279 76.375 0.00819439 0.00940305 2024-03-29T04:38:57.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[102350]: pgmap v14: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-03-29T04:38:57.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:57.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:57.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:57.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:57.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[41899]: pgmap v14: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-03-29T04:38:57.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:57.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:57.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:57.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:56 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:57.614 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:38:57 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:38:58.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:57 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:38:58.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:57 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:38:58.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:57 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:38:58.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 77746 77730 105.595 74.8125 0.0181237 0.00946498 2024-03-29T04:38:58.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:57 smithi029 ceph-mon[102350]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:38:58.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:57 smithi029 ceph-mon[102350]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:38:58.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:57 smithi029 ceph-mon[102350]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:38:58.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:57 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.conf 2024-03-29T04:38:58.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:57 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.conf 2024-03-29T04:38:58.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:57 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:38:59.114 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:38:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: pgmap v15: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:59.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:59.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:59.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 79074 79058 105.114 83 0.0110729 0.00950858 2024-03-29T04:38:59.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:38:59.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:38:59.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:38:59.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: pgmap v15: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: Updating smithi029:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: Updating smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: pgmap v15: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: Updating smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:59.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:59.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:39:00.134 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 80422 80406 104.679 84.25 0.00873046 0.00954809 2024-03-29T04:39:00.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[102350]: pgmap v16: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:39:00.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[102350]: pgmap v17: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-03-29T04:39:00.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[102350]: pgmap v18: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-03-29T04:39:00.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[102350]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:39:00.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[41899]: pgmap v16: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:39:00.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[41899]: pgmap v17: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-03-29T04:39:00.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[41899]: pgmap v18: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-03-29T04:39:00.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:38:59 smithi029 ceph-mon[41899]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:39:00.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:59 smithi099 ceph-mon[33865]: pgmap v16: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:39:00.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:59 smithi099 ceph-mon[33865]: pgmap v17: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-03-29T04:39:00.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:59 smithi099 ceph-mon[33865]: pgmap v18: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-03-29T04:39:00.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:38:59 smithi099 ceph-mon[33865]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:39:01.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 81780 81764 104.275 84.875 0.00516471 0.00958483 2024-03-29T04:39:02.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 83127 83111 103.873 84.1875 0.014579 0.00962191 2024-03-29T04:39:02.166 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:01 smithi099 ceph-mon[33865]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-03-29T04:39:02.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:01 smithi029 ceph-mon[102350]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-03-29T04:39:02.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:01 smithi029 ceph-mon[41899]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-03-29T04:39:03.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 84396 84380 103.391 79.3125 0.00354619 0.00966443 2024-03-29T04:39:04.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 15 85634 85619 102.892 77.4375 0.0171558 0.00971417 2024-03-29T04:39:04.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:03 smithi029 ceph-mon[102350]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-03-29T04:39:04.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:03 smithi029 ceph-mon[41899]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-03-29T04:39:04.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:03 smithi099 ceph-mon[33865]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-03-29T04:39:05.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 86905 86889 102.448 79.375 0.00472859 0.00975588 2024-03-29T04:39:05.184 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:39:04] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.43.0" 2024-03-29T04:39:05.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:39:05.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:05.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-mon[102350]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-03-29T04:39:05.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:39:05.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:05.185 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:04 smithi029 ceph-mon[41899]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-03-29T04:39:05.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:39:05.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:05.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:04 smithi099 ceph-mon[33865]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 945 op/s 2024-03-29T04:39:06.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 88196 88180 102.045 80.6875 0.00773626 0.0097942 2024-03-29T04:39:07.135 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 89438 89422 101.601 77.625 0.0240193 0.0098367 2024-03-29T04:39:07.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:07 smithi099 ceph-mon[33865]: pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-03-29T04:39:07.865 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:39:07 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 10. 2024-03-29T04:39:07.865 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:39:07 smithi099 systemd[1]: Stopped Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:39:07.865 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:39:07 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Start request repeated too quickly. 2024-03-29T04:39:07.865 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:39:07 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:39:07.865 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Mar 29 04:39:07 smithi099 systemd[1]: Failed to start Ceph grafana.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:39:07.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:07 smithi029 ceph-mon[102350]: pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-03-29T04:39:07.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:07 smithi029 ceph-mon[41899]: pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-03-29T04:39:08.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 90679 90663 101.172 77.5625 0.0121406 0.00987906 2024-03-29T04:39:09.115 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:39:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:39:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:39:09.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 91920 91904 100.758 77.5625 0.00673014 0.00991972 2024-03-29T04:39:09.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:09 smithi099 ceph-mon[33865]: pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-03-29T04:39:09.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:09 smithi029 ceph-mon[102350]: pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-03-29T04:39:09.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:09 smithi029 ceph-mon[41899]: pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-03-29T04:39:10.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 93154 93138 100.35 77.125 0.00991987 0.00996014 2024-03-29T04:39:11.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 94371 94355 99.9383 76.0625 0.0206233 0.0100014 2024-03-29T04:39:11.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:11 smithi099 ceph-mon[33865]: pgmap v24: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:39:11.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:11 smithi029 ceph-mon[102350]: pgmap v24: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:39:11.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:11 smithi029 ceph-mon[41899]: pgmap v24: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:39:12.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:39:12.144784+0000 min lat: 0.00132983 max lat: 0.583804 avg lat: 0.010058 2024-03-29T04:39:12.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:39:12.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 95427 95411 99.3726 66 0.0115132 0.010058 2024-03-29T04:39:13.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 96528 96512 98.8715 68.8125 0.0134227 0.0101075 2024-03-29T04:39:13.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:13 smithi099 ceph-mon[33865]: pgmap v25: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:39:13.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:13 smithi029 ceph-mon[102350]: pgmap v25: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:39:13.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:13 smithi029 ceph-mon[41899]: pgmap v25: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:39:14.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 97695 97679 98.4531 72.9375 0.00719572 0.0101522 2024-03-29T04:39:15.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 98841 98825 98.0272 71.625 0.0142397 0.0101967 2024-03-29T04:39:15.184 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:39:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:39:14] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.43.0" 2024-03-29T04:39:15.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:15 smithi099 ceph-mon[33865]: pgmap v26: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:39:15.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:15 smithi029 ceph-mon[102350]: pgmap v26: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:39:15.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:15 smithi029 ceph-mon[41899]: pgmap v26: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:39:16.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 99981 99965 97.6087 71.25 0.00433368 0.0102394 2024-03-29T04:39:17.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 101129 101113 97.2108 71.75 0.0124613 0.0102813 2024-03-29T04:39:17.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:17 smithi099 ceph-mon[33865]: pgmap v27: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-03-29T04:39:17.934 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:17 smithi029 ceph-mon[102350]: pgmap v27: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-03-29T04:39:17.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:17 smithi029 ceph-mon[41899]: pgmap v27: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-03-29T04:39:18.136 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 102285 102269 96.8325 72.25 0.0117765 0.010322 2024-03-29T04:39:19.115 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:39:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:39:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:39:19.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 103403 103387 96.4301 69.875 0.0163782 0.0103652 2024-03-29T04:39:19.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:19 smithi099 ceph-mon[33865]: pgmap v28: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-03-29T04:39:19.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:19.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:19 smithi029 ceph-mon[102350]: pgmap v28: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-03-29T04:39:19.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:19.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:19 smithi029 ceph-mon[41899]: pgmap v28: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-03-29T04:39:19.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:19 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:20.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 104529 104513 96.0468 70.375 0.0138552 0.0104065 2024-03-29T04:39:21.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 105638 105622 95.6593 69.3125 0.00968191 0.0104487 2024-03-29T04:39:21.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:21 smithi029 ceph-mon[102350]: pgmap v29: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-03-29T04:39:21.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:21 smithi029 ceph-mon[41899]: pgmap v29: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-03-29T04:39:21.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:21 smithi099 ceph-mon[33865]: pgmap v29: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-03-29T04:39:22.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 106652 106636 95.1981 63.375 0.0066722 0.0104989 2024-03-29T04:39:23.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 107589 107573 94.682 58.5625 0.021083 0.0105559 2024-03-29T04:39:23.734 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:23 smithi029 ceph-mon[102350]: pgmap v30: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-03-29T04:39:23.734 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:23 smithi029 ceph-mon[41899]: pgmap v30: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-03-29T04:39:23.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:23 smithi099 ceph-mon[33865]: pgmap v30: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-03-29T04:39:24.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 108595 108579 94.2402 62.875 0.0116516 0.0106065 2024-03-29T04:39:25.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 109685 109669 93.8824 68.125 0.0172517 0.0106468 2024-03-29T04:39:25.185 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:39:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:39:24] "GET /metrics HTTP/1.1" 200 36254 "" "Prometheus/2.43.0" 2024-03-29T04:39:25.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:25 smithi099 ceph-mon[33865]: pgmap v31: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-03-29T04:39:25.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:25 smithi029 ceph-mon[102350]: pgmap v31: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-03-29T04:39:25.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:25 smithi029 ceph-mon[41899]: pgmap v31: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-03-29T04:39:26.137 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 110744 110728 93.508 66.1875 0.0241206 0.0106889 2024-03-29T04:39:27.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 111813 111797 93.152 66.8125 0.0245619 0.0107299 2024-03-29T04:39:27.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:27 smithi099 ceph-mon[33865]: pgmap v32: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-03-29T04:39:27.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:27 smithi029 ceph-mon[102350]: pgmap v32: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-03-29T04:39:27.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:27 smithi029 ceph-mon[41899]: pgmap v32: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-03-29T04:39:28.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 112847 112831 92.7766 64.625 0.0085732 0.0107733 2024-03-29T04:39:29.115 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:39:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:39:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:39:29.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 113897 113881 92.4239 65.625 0.012005 0.0108146 2024-03-29T04:39:29.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:29 smithi099 ceph-mon[33865]: pgmap v33: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-03-29T04:39:29.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:29 smithi029 ceph-mon[102350]: pgmap v33: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-03-29T04:39:29.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:29 smithi029 ceph-mon[41899]: pgmap v33: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-03-29T04:39:30.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 114920 114904 92.0587 63.9375 0.0177133 0.0108577 2024-03-29T04:39:31.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 115952 115936 91.7098 64.5 0.0143912 0.0108984 2024-03-29T04:39:31.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:31 smithi099 ceph-mon[33865]: pgmap v34: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-03-29T04:39:31.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:31 smithi029 ceph-mon[102350]: pgmap v34: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-03-29T04:39:31.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:31 smithi029 ceph-mon[41899]: pgmap v34: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-03-29T04:39:32.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:39:32.146605+0000 min lat: 0.00132983 max lat: 0.583804 avg lat: 0.0109417 2024-03-29T04:39:32.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:39:32.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 116961 116945 91.3516 63.0625 0.008211 0.0109417 2024-03-29T04:39:33.138 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 117970 117954 91.0022 63.0625 0.00619811 0.0109814 2024-03-29T04:39:33.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:33 smithi099 ceph-mon[33865]: pgmap v35: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-03-29T04:39:33.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:33 smithi029 ceph-mon[102350]: pgmap v35: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-03-29T04:39:33.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:33 smithi029 ceph-mon[41899]: pgmap v35: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-03-29T04:39:34.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 118951 118935 90.6401 61.3125 0.0184758 0.0110276 2024-03-29T04:39:34.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:34.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:34.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:34 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:35.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 119954 119938 90.3033 62.6875 0.012297 0.0110689 2024-03-29T04:39:35.185 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:39:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:39:34] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-03-29T04:39:35.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:35 smithi099 ceph-mon[33865]: pgmap v36: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-03-29T04:39:35.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:35 smithi029 ceph-mon[102350]: pgmap v36: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-03-29T04:39:35.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:35 smithi029 ceph-mon[41899]: pgmap v36: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s 2024-03-29T04:39:36.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 120954 120938 89.9722 62.5 0.023957 0.0111094 2024-03-29T04:39:37.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 121955 121939 89.6497 62.5625 0.0243149 0.0111493 2024-03-29T04:39:37.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:37 smithi099 ceph-mon[33865]: pgmap v37: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-03-29T04:39:37.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:37 smithi029 ceph-mon[102350]: pgmap v37: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-03-29T04:39:37.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:37 smithi029 ceph-mon[41899]: pgmap v37: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-03-29T04:39:38.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 122941 122925 89.3238 61.625 0.0191093 0.0111901 2024-03-29T04:39:39.115 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:39:38 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:39:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:39:39.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 123602 123586 88.7719 41.3125 0.00607014 0.0112595 2024-03-29T04:39:39.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:39 smithi099 ceph-mon[33865]: pgmap v38: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2024-03-29T04:39:39.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:39 smithi029 ceph-mon[102350]: pgmap v38: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2024-03-29T04:39:39.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:39 smithi029 ceph-mon[41899]: pgmap v38: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2024-03-29T04:39:40.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 124539 124523 88.4286 58.5625 0.0039189 0.0112968 2024-03-29T04:39:41.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 125488 125472 88.1014 59.3125 0.0101495 0.0113454 2024-03-29T04:39:41.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:41 smithi099 ceph-mon[33865]: pgmap v39: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:39:41.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:41 smithi029 ceph-mon[102350]: pgmap v39: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:39:41.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:41 smithi029 ceph-mon[41899]: pgmap v39: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:39:42.139 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0095 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 126456 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 87.8074 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 36.6195 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 220.125 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 41.3125 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1404 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 585.912 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3522 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 661 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0113843 2024-03-29T04:39:42.140 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.013797 2024-03-29T04:39:42.141 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 0.583804 2024-03-29T04:39:42.141 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00132983 2024-03-29T04:39:42.144 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-03-29T04:39:43.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:43 smithi099 ceph-mon[33865]: pgmap v40: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-03-29T04:39:43.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:43 smithi029 ceph-mon[102350]: pgmap v40: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-03-29T04:39:43.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:43 smithi029 ceph-mon[41899]: pgmap v40: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-03-29T04:39:45.185 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:39:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:39:44] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-03-29T04:39:45.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:45 smithi099 ceph-mon[33865]: pgmap v41: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-03-29T04:39:45.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:45 smithi029 ceph-mon[102350]: pgmap v41: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-03-29T04:39:45.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:45 smithi029 ceph-mon[41899]: pgmap v41: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 805 op/s 2024-03-29T04:39:47.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:47 smithi099 ceph-mon[33865]: pgmap v42: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 62 MiB/s wr, 1.21k op/s 2024-03-29T04:39:47.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:47 smithi029 ceph-mon[102350]: pgmap v42: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 62 MiB/s wr, 1.21k op/s 2024-03-29T04:39:47.935 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:47 smithi029 ceph-mon[41899]: pgmap v42: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 62 MiB/s wr, 1.21k op/s 2024-03-29T04:39:49.115 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:39:48 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:39:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:39:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:49 smithi099 ceph-mon[33865]: pgmap v43: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 785 op/s 2024-03-29T04:39:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:49.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:49 smithi029 ceph-mon[102350]: pgmap v43: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 785 op/s 2024-03-29T04:39:49.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:49.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:49 smithi029 ceph-mon[41899]: pgmap v43: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 785 op/s 2024-03-29T04:39:49.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:49 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:51.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:51 smithi099 ceph-mon[33865]: pgmap v44: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 1.20k op/s 2024-03-29T04:39:51.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:51 smithi029 ceph-mon[102350]: pgmap v44: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 1.20k op/s 2024-03-29T04:39:51.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:51 smithi029 ceph-mon[41899]: pgmap v44: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 1.20k op/s 2024-03-29T04:39:53.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:53 smithi099 ceph-mon[33865]: pgmap v45: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 817 op/s 2024-03-29T04:39:53.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:53 smithi029 ceph-mon[102350]: pgmap v45: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 817 op/s 2024-03-29T04:39:53.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:53 smithi029 ceph-mon[41899]: pgmap v45: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 817 op/s 2024-03-29T04:39:55.185 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:39:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:39:54] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-03-29T04:39:55.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:55 smithi099 ceph-mon[33865]: pgmap v46: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 817 op/s 2024-03-29T04:39:55.935 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:55 smithi029 ceph-mon[102350]: pgmap v46: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 817 op/s 2024-03-29T04:39:55.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:55 smithi029 ceph-mon[41899]: pgmap v46: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 817 op/s 2024-03-29T04:39:56.540 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:39:57.026 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:39:57.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:57 smithi099 ceph-mon[33865]: pgmap v47: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 1.22k op/s 2024-03-29T04:39:57.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:57 smithi029 ceph-mon[102350]: pgmap v47: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 1.22k op/s 2024-03-29T04:39:57.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:57 smithi029 ceph-mon[41899]: pgmap v47: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 1.22k op/s 2024-03-29T04:39:58.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:58.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:58 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:58.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:58.866 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:39:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:39:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:39:59.616 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:39:59.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:39:59 smithi099 ceph-mon[33865]: pgmap v48: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-03-29T04:39:59.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:39:59 smithi029 ceph-mon[102350]: pgmap v48: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-03-29T04:39:59.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:39:59 smithi029 ceph-mon[41899]: pgmap v48: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-03-29T04:40:00.355 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for majority of mons to upgrade' 2024-03-29T04:40:00.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:00 smithi099 ceph-mon[33865]: from='client.25280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:00.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:00 smithi099 ceph-mon[33865]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:00.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:00 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:00.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:00 smithi099 ceph-mon[33865]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:40:00.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[102350]: from='client.25280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:00.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[102350]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:00.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:00.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[102350]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:40:00.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[41899]: from='client.25280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:00.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[41899]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:00.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:00.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:00 smithi029 ceph-mon[41899]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:40:01.871 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:01.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:01 smithi029 ceph-mon[102350]: pgmap v49: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-03-29T04:40:01.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:01 smithi029 ceph-mon[41899]: pgmap v49: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-03-29T04:40:02.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:01 smithi099 ceph-mon[33865]: pgmap v49: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-03-29T04:40:03.712 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:03 smithi029 ceph-mon[102350]: pgmap v50: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-03-29T04:40:03.712 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:03 smithi029 ceph-mon[41899]: pgmap v50: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-03-29T04:40:04.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:03 smithi099 ceph-mon[33865]: pgmap v50: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-03-29T04:40:04.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:04.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:04.898 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:04.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:04.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:04.899 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:05.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:05.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:05.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:05.186 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:40:04] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-03-29T04:40:05.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:05 smithi029 ceph-mon[102350]: pgmap v51: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-03-29T04:40:05.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:05 smithi029 ceph-mon[41899]: pgmap v51: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-03-29T04:40:06.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:05 smithi099 ceph-mon[33865]: pgmap v51: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-03-29T04:40:07.679 INFO:teuthology.orchestra.run.smithi029.stdout:wait for majority of mons to upgrade 2024-03-29T04:40:07.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:07 smithi029 ceph-mon[102350]: pgmap v52: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 1.13k op/s 2024-03-29T04:40:07.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:07 smithi029 ceph-mon[102350]: mgrmap e41: y(active, since 92s), standbys: x 2024-03-29T04:40:07.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:07 smithi029 ceph-mon[41899]: pgmap v52: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 1.13k op/s 2024-03-29T04:40:07.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:07 smithi029 ceph-mon[41899]: mgrmap e41: y(active, since 92s), standbys: x 2024-03-29T04:40:08.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:07 smithi099 ceph-mon[33865]: pgmap v52: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 1.13k op/s 2024-03-29T04:40:08.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:07 smithi099 ceph-mon[33865]: mgrmap e41: y(active, since 92s), standbys: x 2024-03-29T04:40:08.441 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-03-29T04:40:09.116 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:40:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:40:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:40:09.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:09 smithi029 ceph-mon[102350]: pgmap v53: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 727 op/s 2024-03-29T04:40:09.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:09 smithi029 ceph-mon[41899]: pgmap v53: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 727 op/s 2024-03-29T04:40:10.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:09 smithi099 ceph-mon[33865]: pgmap v53: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 727 op/s 2024-03-29T04:40:11.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:11 smithi029 ceph-mon[102350]: pgmap v54: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 1.06k op/s 2024-03-29T04:40:11.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:11 smithi029 ceph-mon[41899]: pgmap v54: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 1.06k op/s 2024-03-29T04:40:12.099 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:12.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:11 smithi099 ceph-mon[33865]: pgmap v54: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 1.06k op/s 2024-03-29T04:40:14.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:13 smithi029 ceph-mon[102350]: pgmap v55: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 687 op/s 2024-03-29T04:40:14.089 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:13 smithi029 ceph-mon[41899]: pgmap v55: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 687 op/s 2024-03-29T04:40:14.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:13 smithi099 ceph-mon[33865]: pgmap v55: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 687 op/s 2024-03-29T04:40:14.936 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:40:14] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-03-29T04:40:15.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:15 smithi099 ceph-mon[33865]: pgmap v56: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 687 op/s 2024-03-29T04:40:15.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:15 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:15.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:15 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:15.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:15 smithi029 ceph-mon[102350]: pgmap v56: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 687 op/s 2024-03-29T04:40:15.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:15 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:15.936 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:15 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:15.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:15 smithi029 ceph-mon[41899]: pgmap v56: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 255 B/s wr, 687 op/s 2024-03-29T04:40:15.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:15 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:15.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:15 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:16.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: pgmap v57: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 272 B/s wr, 1.08k op/s 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1227452827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[102350]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:16.843 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: pgmap v57: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 272 B/s wr, 1.08k op/s 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1227452827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:40:16.844 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:16 smithi029 ceph-mon[41899]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:40:16.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:16.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: pgmap v57: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 272 B/s wr, 1.08k op/s 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1227452827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:40:16.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:16 smithi099 ceph-mon[33865]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:40:19.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:18 smithi099 ceph-mon[33865]: pgmap v58: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-03-29T04:40:19.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:18 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3526132511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:19.117 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:40:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:40:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:40:19.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:18 smithi029 ceph-mon[102350]: pgmap v58: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-03-29T04:40:19.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:18 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3526132511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:19.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:18 smithi029 ceph-mon[41899]: pgmap v58: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-03-29T04:40:19.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:18 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/3526132511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:20.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: Upgrade: Updating mon.c 2024-03-29T04:40:20.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:20.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:40:20.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:40:20.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:20.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: Deploying daemon mon.c on smithi029 2024-03-29T04:40:20.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: Upgrade: Updating mon.c 2024-03-29T04:40:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:40:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:40:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:20.172 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: Deploying daemon mon.c on smithi029 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: Upgrade: Updating mon.c 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: Deploying daemon mon.c on smithi029 2024-03-29T04:40:20.173 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:19 smithi029 ceph-mon[41899]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:21.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:20 smithi029 ceph-mon[102350]: pgmap v59: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-03-29T04:40:21.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:20 smithi029 ceph-mon[41899]: pgmap v59: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-03-29T04:40:21.367 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:20 smithi099 ceph-mon[33865]: pgmap v59: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-03-29T04:40:21.889 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:21 smithi029 systemd[1]: Stopping Ceph mon.c for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:40:22.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1084023386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:22.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:21 smithi029 ceph-mon[41899]: from='client.? 172.21.15.29:0/1084023386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:22.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:21 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c[41876]: 2024-03-29T04:40:21.922+0000 7f890948a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:40:22.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:21 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c[41876]: 2024-03-29T04:40:21.922+0000 7f890948a700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-03-29T04:40:22.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:22 smithi029 podman[106420]: 2024-03-29 04:40:22.008975214 +0000 UTC m=+0.221877546 container died 0159b319c9a57c9626417cca5eea6936443cb5ce21f611579877be1d6b215b29 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-29T04:40:22.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:21 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/1084023386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:22.505 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:22 smithi029 podman[106420]: 2024-03-29 04:40:22.496579984 +0000 UTC m=+0.709482290 container cleanup 0159b319c9a57c9626417cca5eea6936443cb5ce21f611579877be1d6b215b29 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326) 2024-03-29T04:40:22.505 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:22 smithi029 bash[106420]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c 2024-03-29T04:40:22.936 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:22 smithi029 podman[106435]: 2024-03-29 04:40:22.835721234 +0000 UTC m=+0.828724051 container remove 0159b319c9a57c9626417cca5eea6936443cb5ce21f611579877be1d6b215b29 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326) 2024-03-29T04:40:23.425 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:23 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.c.service: Deactivated successfully. 2024-03-29T04:40:23.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:23 smithi029 systemd[1]: Stopped Ceph mon.c for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:40:23.426 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:23 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.c.service: Consumed 9.759s CPU time. 2024-03-29T04:40:23.686 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:23 smithi029 systemd[1]: Starting Ceph mon.c for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:40:24.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:23 smithi029 podman[106585]: 2024-03-29 04:40:23.784798079 +0000 UTC m=+0.055597919 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:24.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:23 smithi029 podman[106585]: 2024-03-29 04:40:23.915272339 +0000 UTC m=+0.186072170 container create 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 podman[106585]: 2024-03-29 04:40:24.279884569 +0000 UTC m=+0.550684381 container init 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 podman[106585]: 2024-03-29 04:40:24.294776915 +0000 UTC m=+0.565576709 container start 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 6 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: pidfile_write: ignore empty --pid-file 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: load: jerasure load: lrc 2024-03-29T04:40:24.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Git sha 0 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: DB SUMMARY 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: DB Session ID: YU16IXSTAK4EJS70YCBM 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1082 Bytes 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 7695730 ; 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:40:24.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.env: 0x557ef27cbc60 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.info_log: 0x557ef3ec5ae0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.statistics: (nil) 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.use_fsync: 0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:40:24.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.db_log_dir: 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.wal_dir: 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.write_buffer_manager: 0x557ef3e981e0 2024-03-29T04:40:24.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.unordered_write: 0 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.row_cache: None 2024-03-29T04:40:24.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.wal_filter: None 2024-03-29T04:40:24.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:40:24.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:40:24.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:40:24.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:40:24.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.wal_compression: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:40:24.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_open_files: -1 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:40:24.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Compression algorithms supported: 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kZSTD supported: 0 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.merge_operator: 2024-03-29T04:40:24.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_filter: None 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557ef3ec5be0) 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_type: 0 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_index_type: 0 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_shortening: 1 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: checksum: 4 2024-03-29T04:40:24.698 INFO:journalctl@ceph.mon.c.smithi029.stdout: no_block_cache: 0 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache: 0x557ef3ec9090 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_options: 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: capacity : 536870912 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_shard_bits : 4 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: strict_capacity_limit : 0 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_cache_compressed: (nil) 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: persistent_cache: (nil) 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size: 4096 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_size_deviation: 10 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_restart_interval: 16 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: index_block_restart_interval: 1 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: metadata_block_size: 4096 2024-03-29T04:40:24.699 INFO:journalctl@ceph.mon.c.smithi029.stdout: partition_filters: 0 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: use_delta_encoding: 1 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: filter_policy: bloomfilter 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: whole_key_filtering: 1 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: verify_compression: 0 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: format_version: 5 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: enable_index_compression: 1 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: block_align: 0 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: max_auto_readahead_size: 262144 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: prepopulate_block_cache: 0 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:40:24.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression: NoCompression 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.num_levels: 7 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:40:24.701 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:40:24.702 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:40:24.703 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:40:24.704 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.ttl: 2592000 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:40:24.705 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12662, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 78409c6f-4e62-4224-ad72-b11bdd045aa1 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687224339476, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-03-29T04:40:24.706 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687224364265, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4738721, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12667, "largest_seqno": 15450, "table_properties": {"data_size": 4727243, "index_size": 7321, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3141, "raw_key_size": 26533, "raw_average_key_size": 21, "raw_value_size": 4702587, "raw_average_value_size": 3786, "num_data_blocks": 330, "num_entries": 1242, "num_filter_entries": 1242, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711687224, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "78409c6f-4e62-4224-ad72-b11bdd045aa1", "db_session_id": "YU16IXSTAK4EJS70YCBM", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687224364407, "job": 1, "event": "recovery_finished"} 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557ef3fc8000 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: rocksdb: DB pointer 0x557ef3fb6000 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-mon[106624]: starting mon.c rank 2 at public addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] at bind addrs [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 bash[106585]: 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e 2024-03-29T04:40:24.707 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:24 smithi029 systemd[1]: Started Ceph mon.c for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:40:25.186 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:40:24] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2024-03-29T04:40:29.116 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:40:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:40:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:40:29.686 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:29 smithi029 ceph-mon[102350]: pgmap v60: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:40:29.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:29 smithi099 ceph-mon[33865]: pgmap v60: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: mon.c calling monitor election 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: mon.a calling monitor election 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: mon.b calling monitor election 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: monmap epoch 3 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: last_changed 2024-03-29T04:26:59.304920+0000 2024-03-29T04:40:30.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: min_mon_release 17 (quincy) 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: election_strategy: 1 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: mgrmap e41: y(active, since 116s), standbys: x 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:40:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[102350]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: pgmap v60: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: mon.c calling monitor election 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: mon.a calling monitor election 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: mon.b calling monitor election 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: monmap epoch 3 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: last_changed 2024-03-29T04:26:59.304920+0000 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:40:30.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: min_mon_release 17 (quincy) 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: election_strategy: 1 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: mgrmap e41: y(active, since 116s), standbys: x 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:40:30.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:30 smithi029 ceph-mon[106624]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: mon.c calling monitor election 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: mon.a calling monitor election 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: mon.b calling monitor election 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: monmap epoch 3 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: last_changed 2024-03-29T04:26:59.304920+0000 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: min_mon_release 17 (quincy) 2024-03-29T04:40:30.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: election_strategy: 1 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: mgrmap e41: y(active, since 116s), standbys: x 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:40:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:30 smithi099 ceph-mon[33865]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:40:32.797 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 2 2024-03-29T04:40:33.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:32 smithi099 ceph-mon[33865]: pgmap v65: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:40:33.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:32 smithi029 ceph-mon[102350]: pgmap v65: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:40:33.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:32 smithi029 ceph-mon[106624]: pgmap v65: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:40:34.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:33 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/3554981271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:34.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3554981271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:34.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3554981271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:40:34.686 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:40:34.381+0000 7f94b7a0c700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-29T04:40:35.088 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:40:34] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.43.0" 2024-03-29T04:40:35.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:34 smithi029 ceph-mon[106624]: pgmap v66: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-03-29T04:40:35.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:35.088 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:34 smithi029 ceph-mon[102350]: pgmap v66: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-03-29T04:40:35.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:34 smithi099 ceph-mon[33865]: pgmap v66: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2024-03-29T04:40:35.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:35.482 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:40:36.208 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:37.116 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:36 smithi099 ceph-mon[33865]: pgmap v67: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-03-29T04:40:37.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:36 smithi029 ceph-mon[102350]: pgmap v67: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-03-29T04:40:37.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:36 smithi029 ceph-mon[106624]: pgmap v67: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-03-29T04:40:39.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:38 smithi099 ceph-mon[33865]: pgmap v68: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:39.117 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:40:38 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:40:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:40:39.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:38 smithi029 ceph-mon[102350]: pgmap v68: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:39.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:38 smithi029 ceph-mon[106624]: pgmap v68: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:41.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:40 smithi099 ceph-mon[33865]: pgmap v69: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:41.186 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:40 smithi029 ceph-mon[102350]: pgmap v69: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:41.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:40 smithi029 ceph-mon[106624]: pgmap v69: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:42.224 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:40:42.224 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (4m) 27s ago 9m 24.2M - 0.25.0 c8568f914cd2 6fb7bc148154 2024-03-29T04:40:42.224 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi099 *:3000 error 38s ago 9m - - 2024-03-29T04:40:42.224 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.gsovzu smithi029 running (7m) 27s ago 7m 15.7M - 17.2.7-789-g4213e17c cf22eb8683be e6b24be4eed9 2024-03-29T04:40:42.224 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi099.jhvkhg smithi099 running (7m) 38s ago 7m 16.5M - 17.2.7-789-g4213e17c cf22eb8683be 2d2ff55bb03a 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi099 *:8443,9283,8765 running (2m) 38s ago 13m 450M - 19.0.0-1959-g8da23624 54112f57da44 1c6cc4c8d8c3 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (3m) 27s ago 15m 498M - 19.0.0-1959-g8da23624 54112f57da44 280a22bdfead 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (2m) 27s ago 15m 41.8M 2048M 19.0.0-1959-g8da23624 54112f57da44 4ef356385530 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi099 running (13m) 38s ago 13m 54.3M 2048M 17.2.7-789-g4213e17c cf22eb8683be 48c38660bc12 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 starting - - - 2048M 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (4m) 27s ago 8m 16.5M - 1.5.0 0da6a335fe13 9026c9798e07 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi099 *:9100 running (3m) 38s ago 8m 15.9M - 1.5.0 0da6a335fe13 52f151f28461 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (13m) 27s ago 13m 370M 2481M 17.2.7-789-g4213e17c cf22eb8683be 5e5c90bac829 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (12m) 27s ago 12m 320M 2481M 17.2.7-789-g4213e17c cf22eb8683be 6532f8ba40e9 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (12m) 27s ago 12m 446M 2481M 17.2.7-789-g4213e17c cf22eb8683be 817a836d7edf 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (11m) 27s ago 11m 463M 2481M 17.2.7-789-g4213e17c cf22eb8683be a4cb24ddb406 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi099 running (11m) 38s ago 11m 392M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1fab66997673 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi099 running (11m) 38s ago 11m 421M 2480M 17.2.7-789-g4213e17c cf22eb8683be 78420bf6aaa9 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi099 running (10m) 38s ago 10m 291M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1f39d06fae80 2024-03-29T04:40:42.225 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi099 running (10m) 38s ago 10m 370M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6adf0209fbb4 2024-03-29T04:40:42.226 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi099 *:9095 running (2m) 38s ago 9m 56.5M - 2.43.0 a07b618ecd1d cfdb8be5d010 2024-03-29T04:40:42.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:42 smithi029 ceph-mon[102350]: pgmap v70: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:40:42.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:42 smithi029 ceph-mon[106624]: pgmap v70: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:40:42.989 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:40:43.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:42 smithi099 ceph-mon[33865]: pgmap v70: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:40:44.005 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:44.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:43 smithi099 ceph-mon[33865]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:44.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:43 smithi029 ceph-mon[102350]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:44.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:43 smithi029 ceph-mon[106624]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:45.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-mon[102350]: pgmap v71: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-03-29T04:40:45.047 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:45.048 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:45.048 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:40:44] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.43.0" 2024-03-29T04:40:45.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-mon[106624]: pgmap v71: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-03-29T04:40:45.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:45.048 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:44 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:45.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:44 smithi099 ceph-mon[33865]: pgmap v71: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-03-29T04:40:45.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:44 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:45.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:44 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:46.675 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:46.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[102350]: pgmap v72: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-03-29T04:40:46.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:46.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:46.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:46.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[106624]: pgmap v72: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-03-29T04:40:46.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:46.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:46.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:47.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:46 smithi099 ceph-mon[33865]: pgmap v72: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2024-03-29T04:40:47.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:46 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:47.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:46 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:47.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:46 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:47.426 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:40:47.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:47 smithi029 ceph-mon[102350]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:47.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:47 smithi029 ceph-mon[102350]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:47.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:47 smithi029 ceph-mon[106624]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:47.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:47 smithi029 ceph-mon[106624]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:48.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:47 smithi099 ceph-mon[33865]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:40:48.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:47 smithi099 ceph-mon[33865]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:40:48.862 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:49.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:48 smithi099 ceph-mon[33865]: pgmap v73: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-03-29T04:40:49.117 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:40:48 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:40:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:40:49.141 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:48 smithi029 ceph-mon[102350]: pgmap v73: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-03-29T04:40:49.141 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:48 smithi029 ceph-mon[106624]: pgmap v73: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-03-29T04:40:50.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:50.187 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:50.187 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:50.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:50 smithi029 ceph-mon[102350]: pgmap v74: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-03-29T04:40:50.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:50 smithi029 ceph-mon[106624]: pgmap v74: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-03-29T04:40:51.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:50 smithi099 ceph-mon[33865]: pgmap v74: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-03-29T04:40:52.933 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:52 smithi029 ceph-mon[102350]: pgmap v75: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-03-29T04:40:52.933 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:52 smithi029 ceph-mon[106624]: pgmap v75: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-03-29T04:40:53.117 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:52 smithi099 ceph-mon[33865]: pgmap v75: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: pgmap v76: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:54.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:54.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:54.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: pgmap v77: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-03-29T04:40:54.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:54 smithi099 ceph-mon[33865]: pgmap v78: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-03-29T04:40:54.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: pgmap v76: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: pgmap v77: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[102350]: pgmap v78: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:54.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: pgmap v76: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: pgmap v77: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:40:54.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-mon[106624]: pgmap v78: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-03-29T04:40:55.187 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:40:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:40:54] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2024-03-29T04:40:56.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:56 smithi099 ceph-mon[33865]: pgmap v79: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2024-03-29T04:40:56.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:56 smithi029 ceph-mon[102350]: pgmap v79: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2024-03-29T04:40:56.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:56 smithi029 ceph-mon[106624]: pgmap v79: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2024-03-29T04:40:58.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:40:58 smithi099 ceph-mon[33865]: pgmap v80: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-03-29T04:40:58.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:40:58 smithi029 ceph-mon[102350]: pgmap v80: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-03-29T04:40:58.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:40:58 smithi029 ceph-mon[106624]: pgmap v80: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-03-29T04:40:59.117 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:40:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:40:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:41:00.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:00 smithi099 ceph-mon[33865]: pgmap v81: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-03-29T04:41:00.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:00 smithi029 ceph-mon[102350]: pgmap v81: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-03-29T04:41:00.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:00 smithi029 ceph-mon[106624]: pgmap v81: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2024-03-29T04:41:02.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:02 smithi099 ceph-mon[33865]: pgmap v82: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:41:02.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:02 smithi029 ceph-mon[102350]: pgmap v82: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:41:02.687 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:02 smithi029 ceph-mon[106624]: pgmap v82: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:41:04.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:04 smithi029 ceph-mon[102350]: pgmap v83: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-03-29T04:41:04.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:04.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:04 smithi029 ceph-mon[106624]: pgmap v83: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-03-29T04:41:04.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:04.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:04 smithi099 ceph-mon[33865]: pgmap v83: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-03-29T04:41:04.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:05.187 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:41:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:41:04] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.43.0" 2024-03-29T04:41:06.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:06 smithi099 ceph-mon[33865]: pgmap v84: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:41:06.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:06 smithi029 ceph-mon[102350]: pgmap v84: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:41:06.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:06 smithi029 ceph-mon[106624]: pgmap v84: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:41:08.617 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:08 smithi099 ceph-mon[33865]: pgmap v85: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-03-29T04:41:08.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:08 smithi029 ceph-mon[102350]: pgmap v85: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-03-29T04:41:08.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:08 smithi029 ceph-mon[106624]: pgmap v85: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2024-03-29T04:41:09.117 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:41:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:41:08.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:41:10.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:10 smithi029 ceph-mon[102350]: pgmap v86: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-03-29T04:41:10.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:10 smithi029 ceph-mon[106624]: pgmap v86: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-03-29T04:41:10.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:10 smithi099 ceph-mon[33865]: pgmap v86: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2024-03-29T04:41:12.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:12 smithi029 ceph-mon[102350]: pgmap v87: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-03-29T04:41:12.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:12 smithi029 ceph-mon[106624]: pgmap v87: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-03-29T04:41:12.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:12 smithi099 ceph-mon[33865]: pgmap v87: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-03-29T04:41:14.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:14 smithi029 ceph-mon[102350]: pgmap v88: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-03-29T04:41:14.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:14 smithi029 ceph-mon[106624]: pgmap v88: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-03-29T04:41:14.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:14 smithi099 ceph-mon[33865]: pgmap v88: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-03-29T04:41:15.187 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:41:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:41:14] "GET /metrics HTTP/1.1" 200 36269 "" "Prometheus/2.43.0" 2024-03-29T04:41:16.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:16 smithi029 ceph-mon[102350]: pgmap v89: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:41:16.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:16 smithi029 ceph-mon[106624]: pgmap v89: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:41:16.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:16 smithi099 ceph-mon[33865]: pgmap v89: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:41:18.654 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:18 smithi099 ceph-mon[33865]: pgmap v90: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-03-29T04:41:18.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:18 smithi029 ceph-mon[102350]: pgmap v90: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-03-29T04:41:18.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:18 smithi029 ceph-mon[106624]: pgmap v90: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-03-29T04:41:19.117 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:41:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:41:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:41:19.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:19.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:19.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:19 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:20.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:20 smithi029 ceph-mon[102350]: pgmap v91: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-03-29T04:41:20.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:20 smithi029 ceph-mon[106624]: pgmap v91: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-03-29T04:41:20.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:20 smithi099 ceph-mon[33865]: pgmap v91: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-03-29T04:41:22.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:22 smithi029 ceph-mon[102350]: pgmap v92: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-03-29T04:41:22.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:22 smithi029 ceph-mon[106624]: pgmap v92: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-03-29T04:41:22.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:22 smithi099 ceph-mon[33865]: pgmap v92: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-03-29T04:41:24.687 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:24 smithi029 ceph-mon[102350]: pgmap v93: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-03-29T04:41:24.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:24 smithi029 ceph-mon[106624]: pgmap v93: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-03-29T04:41:24.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:24 smithi099 ceph-mon[33865]: pgmap v93: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-03-29T04:41:25.188 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:41:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:41:24] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-03-29T04:41:26.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:26 smithi029 ceph-mon[102350]: pgmap v94: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:41:26.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:26 smithi029 ceph-mon[106624]: pgmap v94: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:41:26.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:26 smithi099 ceph-mon[33865]: pgmap v94: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:41:28.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:28 smithi029 ceph-mon[102350]: pgmap v95: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-03-29T04:41:28.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:28 smithi029 ceph-mon[106624]: pgmap v95: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-03-29T04:41:28.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:28 smithi099 ceph-mon[33865]: pgmap v95: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2024-03-29T04:41:28.868 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:41:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:41:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.29\", device_class=\"ssd\", hostname=\"smithi029\", instance=\"smithi029:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.29\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:41:30.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:30 smithi029 ceph-mon[102350]: pgmap v96: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-03-29T04:41:30.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:30 smithi029 ceph-mon[106624]: pgmap v96: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-03-29T04:41:30.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:30 smithi099 ceph-mon[33865]: pgmap v96: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-03-29T04:41:32.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:32 smithi029 ceph-mon[102350]: pgmap v97: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-03-29T04:41:32.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:32 smithi029 ceph-mon[106624]: pgmap v97: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-03-29T04:41:32.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:32 smithi099 ceph-mon[33865]: pgmap v97: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-03-29T04:41:34.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:34 smithi029 ceph-mon[102350]: pgmap v98: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-03-29T04:41:34.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:34.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:34 smithi029 ceph-mon[106624]: pgmap v98: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-03-29T04:41:34.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:34.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:34 smithi099 ceph-mon[33865]: pgmap v98: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-03-29T04:41:34.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:34 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:35.188 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:41:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:41:34] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.43.0" 2024-03-29T04:41:36.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:36 smithi099 ceph-mon[33865]: pgmap v99: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:41:36.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:36 smithi029 ceph-mon[102350]: pgmap v99: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:41:36.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:36 smithi029 ceph-mon[106624]: pgmap v99: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:41:38.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:38 smithi099 ceph-mon[33865]: pgmap v100: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-03-29T04:41:38.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:38 smithi029 ceph-mon[102350]: pgmap v100: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-03-29T04:41:38.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:38 smithi029 ceph-mon[106624]: pgmap v100: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-03-29T04:41:40.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:40 smithi099 ceph-mon[33865]: pgmap v101: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-03-29T04:41:40.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:40 smithi029 ceph-mon[102350]: pgmap v101: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-03-29T04:41:40.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:40 smithi029 ceph-mon[106624]: pgmap v101: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-03-29T04:41:42.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:42 smithi099 ceph-mon[33865]: pgmap v102: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-03-29T04:41:42.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:42 smithi029 ceph-mon[102350]: pgmap v102: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-03-29T04:41:42.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:42 smithi029 ceph-mon[106624]: pgmap v102: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-03-29T04:41:44.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:44 smithi099 ceph-mon[33865]: pgmap v103: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:44.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:44 smithi029 ceph-mon[102350]: pgmap v103: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:44.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:44 smithi029 ceph-mon[106624]: pgmap v103: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:45.188 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:41:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:41:44] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.43.0" 2024-03-29T04:41:46.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:46 smithi099 ceph-mon[33865]: pgmap v104: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:41:46.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:46 smithi029 ceph-mon[102350]: pgmap v104: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:41:46.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:46 smithi029 ceph-mon[106624]: pgmap v104: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-03-29T04:41:48.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:48 smithi099 ceph-mon[33865]: pgmap v105: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-03-29T04:41:48.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:48 smithi029 ceph-mon[102350]: pgmap v105: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-03-29T04:41:48.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:48 smithi029 ceph-mon[106624]: pgmap v105: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-03-29T04:41:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:49 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:49.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:49.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:50.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:50 smithi099 ceph-mon[33865]: pgmap v106: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:50.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:50 smithi029 ceph-mon[102350]: pgmap v106: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:50.938 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:50 smithi029 ceph-mon[106624]: pgmap v106: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:52.377 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:41:52.841 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:52.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:52 smithi099 ceph-mon[33865]: pgmap v107: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-03-29T04:41:52.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:52 smithi029 ceph-mon[102350]: pgmap v107: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-03-29T04:41:52.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:52 smithi029 ceph-mon[106624]: pgmap v107: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-03-29T04:41:53.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:53 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:53.688 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:53 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:53.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:53 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:54.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:54 smithi099 ceph-mon[33865]: pgmap v108: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:54.901 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:54 smithi029 ceph-mon[102350]: pgmap v108: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:54.901 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:54 smithi029 ceph-mon[106624]: pgmap v108: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-03-29T04:41:55.188 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:41:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:41:54] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.43.0" 2024-03-29T04:41:56.618 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:56 smithi099 ceph-mon[33865]: pgmap v109: 89 pgs: 89 active+clean; 848 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:41:56.832 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:56 smithi029 ceph-mon[102350]: pgmap v109: 89 pgs: 89 active+clean; 848 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:41:56.832 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:56 smithi029 ceph-mon[106624]: pgmap v109: 89 pgs: 89 active+clean; 848 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-03-29T04:41:58.093 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:41:58.831 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:58 smithi029 ceph-mon[106624]: pgmap v110: 89 pgs: 89 active+clean; 803 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2024-03-29T04:41:58.831 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:58 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:58.832 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:58 smithi029 ceph-mon[102350]: pgmap v110: 89 pgs: 89 active+clean; 803 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2024-03-29T04:41:58.832 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:58.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:58 smithi099 ceph-mon[33865]: pgmap v110: 89 pgs: 89 active+clean; 803 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 769 op/s 2024-03-29T04:41:58.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:58 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:59.234 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for all mons to upgrade' 2024-03-29T04:41:59.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:59 smithi099 ceph-mon[33865]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:41:59.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:59 smithi099 ceph-mon[33865]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:41:59.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:59 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:59.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:41:59 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[102350]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[102350]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[106624]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[106624]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:41:59.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:41:59 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:00.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:00 smithi099 ceph-mon[33865]: pgmap v111: 89 pgs: 89 active+clean; 803 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-03-29T04:42:00.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:00 smithi029 ceph-mon[102350]: pgmap v111: 89 pgs: 89 active+clean; 803 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-03-29T04:42:00.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:00 smithi029 ceph-mon[106624]: pgmap v111: 89 pgs: 89 active+clean; 803 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-03-29T04:42:01.310 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:02.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:02 smithi099 ceph-mon[33865]: pgmap v112: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:42:02.938 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:02 smithi029 ceph-mon[102350]: pgmap v112: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:42:02.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:02 smithi029 ceph-mon[106624]: pgmap v112: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-03-29T04:42:04.578 INFO:teuthology.orchestra.run.smithi029.stdout:wait for all mons to upgrade 2024-03-29T04:42:04.688 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:04 smithi029 ceph-mon[102350]: pgmap v113: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2024-03-29T04:42:04.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:04.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:04 smithi029 ceph-mon[106624]: pgmap v113: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2024-03-29T04:42:04.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:04.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:04 smithi099 ceph-mon[33865]: pgmap v113: 89 pgs: 89 active+clean; 564 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2024-03-29T04:42:04.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:04 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:05.188 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:42:04] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2024-03-29T04:42:05.919 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-03-29T04:42:06.385 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:07.118 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:06 smithi099 ceph-mon[33865]: pgmap v114: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-03-29T04:42:07.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:06 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:07.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:06 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:07.188 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:06 smithi029 ceph-mon[102350]: pgmap v114: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-03-29T04:42:07.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:07.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:07.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:06 smithi029 ceph-mon[106624]: pgmap v114: 89 pgs: 89 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-03-29T04:42:07.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:07.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:08.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:08.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:08.070 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:42:08.071 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:07 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:42:08.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:07 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:42:09.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:08 smithi099 ceph-mon[33865]: pgmap v115: 89 pgs: 89 active+clean; 297 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-03-29T04:42:09.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:08 smithi099 ceph-mon[33865]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:42:09.188 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:08 smithi029 ceph-mon[102350]: pgmap v115: 89 pgs: 89 active+clean; 297 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-03-29T04:42:09.188 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:08 smithi029 ceph-mon[102350]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:42:09.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:08 smithi029 ceph-mon[106624]: pgmap v115: 89 pgs: 89 active+clean; 297 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-03-29T04:42:09.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:08 smithi029 ceph-mon[106624]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:42:10.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:09 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/625336519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:10.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:09 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:10.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:09 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:10.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:09 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:10.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:09 smithi099 ceph-mon[33865]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/625336519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/625336519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:10.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:10.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:10.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:09 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:10.958 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 126456 objects 2024-03-29T04:42:10.959 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :148.811 2024-03-29T04:42:11.118 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:10 smithi099 ceph-mon[33865]: pgmap v116: 89 pgs: 89 active+clean; 297 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-03-29T04:42:11.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:10 smithi099 ceph-mon[33865]: Upgrade: Updating mon.b 2024-03-29T04:42:11.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:10 smithi099 ceph-mon[33865]: Deploying daemon mon.b on smithi099 2024-03-29T04:42:11.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:10 smithi029 ceph-mon[106624]: pgmap v116: 89 pgs: 89 active+clean; 297 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-03-29T04:42:11.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:10 smithi029 ceph-mon[106624]: Upgrade: Updating mon.b 2024-03-29T04:42:11.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:10 smithi029 ceph-mon[106624]: Deploying daemon mon.b on smithi099 2024-03-29T04:42:11.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:10 smithi029 ceph-mon[102350]: pgmap v116: 89 pgs: 89 active+clean; 297 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-03-29T04:42:11.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:10 smithi029 ceph-mon[102350]: Upgrade: Updating mon.b 2024-03-29T04:42:11.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:10 smithi029 ceph-mon[102350]: Deploying daemon mon.b on smithi099 2024-03-29T04:42:12.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:11 smithi099 ceph-mon[33865]: from='client.? 172.21.15.29:0/741164275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:12.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 systemd[1]: Stopping Ceph mon.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:42:12.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:11 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/741164275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:12.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:11 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/741164275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:12.303 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b[33829]: 2024-03-29T04:42:12.286+0000 7f3d7d47e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:42:12.303 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b[33829]: 2024-03-29T04:42:12.286+0000 7f3d7d47e700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-03-29T04:42:12.590 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 podman[93841]: 2024-03-29 04:42:12.307574888 +0000 UTC m=+0.128605293 container died 48c38660bc126f3622d55eb30e8fc187899212ed7d0ed3506868c15be8bb4ec1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-29T04:42:12.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 podman[93841]: 2024-03-29 04:42:12.692658187 +0000 UTC m=+0.513688581 container cleanup 48c38660bc126f3622d55eb30e8fc187899212ed7d0ed3506868c15be8bb4ec1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T04:42:12.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 bash[93841]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b 2024-03-29T04:42:13.250 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:12 smithi099 podman[93860]: 2024-03-29 04:42:12.94417771 +0000 UTC m=+0.636641623 container remove 48c38660bc126f3622d55eb30e8fc187899212ed7d0ed3506868c15be8bb4ec1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS) 2024-03-29T04:42:13.251 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:13 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-29T04:42:13.251 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:13 smithi099 systemd[1]: Stopped Ceph mon.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:42:13.251 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:13 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.b.service: Consumed 11.034s CPU time. 2024-03-29T04:42:13.619 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:13 smithi099 systemd[1]: Starting Ceph mon.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:42:13.619 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:13 smithi099 podman[93998]: 2024-03-29 04:42:13.371152292 +0000 UTC m=+0.018233609 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:42:14.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 podman[93998]: 2024-03-29 04:42:14.004669724 +0000 UTC m=+0.651760018 container create a53b591d5ed70b95a39e768892e628f788199907f5a44105e74403049ad7595a (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 podman[93998]: 2024-03-29 04:42:14.607518014 +0000 UTC m=+1.254599326 container init a53b591d5ed70b95a39e768892e628f788199907f5a44105e74403049ad7595a (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-8da2362) 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 podman[93998]: 2024-03-29 04:42:14.613604547 +0000 UTC m=+1.260685849 container start a53b591d5ed70b95a39e768892e628f788199907f5a44105e74403049ad7595a (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True) 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 7 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: pidfile_write: ignore empty --pid-file 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: load: jerasure load: lrc 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Git sha 0 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: DB SUMMARY 2024-03-29T04:42:14.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: DB Session ID: TJEWWEKUHFBNKBEDN7JX 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1082 Bytes 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9465452 ; 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.env: 0x55fb2f3e4c60 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.info_log: 0x55fb302ddae0 2024-03-29T04:42:14.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:42:14.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.statistics: (nil) 2024-03-29T04:42:14.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.use_fsync: 0 2024-03-29T04:42:14.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:42:14.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:42:14.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:42:14.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:42:14.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.db_log_dir: 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.wal_dir: 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:42:14.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.write_buffer_manager: 0x55fb302b01e0 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:42:14.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.unordered_write: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.row_cache: None 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.wal_filter: None 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.wal_compression: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:42:14.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:42:14.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_open_files: -1 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Compression algorithms supported: 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kZSTD supported: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:42:14.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.merge_operator: 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_filter: None 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:42:14.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fb302ddbe0) 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_type: 0 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_index_type: 0 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_shortening: 1 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: checksum: 4 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: no_block_cache: 0 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache: 0x55fb302e1090 2024-03-29T04:42:14.880 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_options: 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: capacity : 536870912 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_shard_bits : 4 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: strict_capacity_limit : 0 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_compressed: (nil) 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: persistent_cache: (nil) 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size: 4096 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size_deviation: 10 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_restart_interval: 16 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_block_restart_interval: 1 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_block_size: 4096 2024-03-29T04:42:14.881 INFO:journalctl@ceph.mon.b.smithi099.stdout: partition_filters: 0 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: use_delta_encoding: 1 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: filter_policy: bloomfilter 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: whole_key_filtering: 1 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: verify_compression: 0 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: format_version: 5 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_index_compression: 1 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_align: 0 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_auto_readahead_size: 262144 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: prepopulate_block_cache: 0 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:42:14.882 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression: NoCompression 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.num_levels: 7 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:42:14.883 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:42:14.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:42:14.885 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:42:14.886 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.ttl: 2592000 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:42:14.887 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:42:14.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12740, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 79578a7e-8e49-4156-a6e0-c4537304156e 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687334646353, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687334669927, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5808345, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12745, "largest_seqno": 16742, "table_properties": {"data_size": 5792640, "index_size": 10524, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 35043, "raw_average_key_size": 21, "raw_value_size": 5759288, "raw_average_value_size": 3498, "num_data_blocks": 472, "num_entries": 1646, "num_filter_entries": 1646, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711687334, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "79578a7e-8e49-4156-a6e0-c4537304156e", "db_session_id": "TJEWWEKUHFBNKBEDN7JX", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687334670046, "job": 1, "event": "recovery_finished"} 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fb303d2000 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: DB pointer 0x55fb303bc000 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:42:14.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** DB Stats ** 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 5.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 550.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:42:14.890 INFO:journalctl@ceph.mon.b.smithi099.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 2/0 12.06 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 550.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 550.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 550.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:42:14.891 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.01 GB write, 208.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.01 GB write, 208.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: Block cache BinnedLRUCache@0x55fb302e1090#7 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: starting mon.b rank 1 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???) e3 preinit fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).mds e9 new map 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).mds e9 print_map 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: e9 2024-03-29T04:42:14.892 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: legacy client fscid: 1 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: Filesystem 'foo' (1) 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: fs_name foo 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: epoch 8 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: created 2024-03-29T04:32:31.571680+0000 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: modified 2024-03-29T04:32:51.533520+0000 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: tableserver 0 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: root 0 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: session_timeout 60 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: session_autoclose 300 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_file_size 1099511627776 2024-03-29T04:42:14.893 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_xattr_size 65536 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: required_client_features {} 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: last_failure 0 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: last_failure_osd_epoch 0 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_mds 1 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: in 0 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: up {0=14661} 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: failed 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: damaged 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: stopped 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_pools [4] 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_pool 3 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: inline_data disabled 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: balancer 2024-03-29T04:42:14.894 INFO:journalctl@ceph.mon.b.smithi099.stdout: bal_rank_mask -1 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: standby_count_wanted 1 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: [mds.foo.smithi029.gsovzu{0:14661} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.29:6834/873175522,v1:172.21.15.29:6835/873175522] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: Standby daemons: 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout: [mds.foo.smithi099.jhvkhg{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.99:6832/954747986,v1:172.21.15.99:6833/954747986] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 ceph-mon[94036]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 bash[93998]: a53b591d5ed70b95a39e768892e628f788199907f5a44105e74403049ad7595a 2024-03-29T04:42:14.895 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:14 smithi099 systemd[1]: Started Ceph mon.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:42:15.189 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:42:14] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2024-03-29T04:42:16.130 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-03-29T04:42:16.130 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-03-29T04:42:16.156 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:15 smithi029 ceph-mon[102350]: pgmap v117: 89 pgs: 89 active+clean; 47 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:42:16.156 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:15 smithi029 ceph-mon[106624]: pgmap v117: 89 pgs: 89 active+clean; 47 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:42:16.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:15 smithi099 ceph-mon[94036]: pgmap v117: 89 pgs: 89 active+clean; 47 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-03-29T04:42:16.399 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:16.453 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 3 2024-03-29T04:42:17.150 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: pgmap v119: 89 pgs: 89 active+clean; 17 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-03-29T04:42:17.150 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:42:17.150 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: mon.a calling monitor election 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: mon.b calling monitor election 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: mon.c calling monitor election 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: monmap epoch 4 2024-03-29T04:42:17.151 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: last_changed 2024-03-29T04:42:15.884907+0000 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: min_mon_release 19 (squid) 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: election_strategy: 1 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: mgrmap e41: y(active, since 3m), standbys: x 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:17.152 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1045359964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: pgmap v119: 89 pgs: 89 active+clean; 17 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: mon.a calling monitor election 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: mon.b calling monitor election 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: mon.c calling monitor election 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:42:17.153 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: monmap epoch 4 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: last_changed 2024-03-29T04:42:15.884907+0000 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: min_mon_release 19 (squid) 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: election_strategy: 1 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:42:17.154 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: mgrmap e41: y(active, since 3m), standbys: x 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:17.155 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:16 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1045359964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:17.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: pgmap v119: 89 pgs: 89 active+clean; 17 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-03-29T04:42:17.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:42:17.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:42:17.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: mon.a calling monitor election 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: mon.b calling monitor election 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: mon.c calling monitor election 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:42:17.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: monmap epoch 4 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: fsid 37319ec8-ed84-11ee-95d0-87774f69a715 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: last_changed 2024-03-29T04:42:15.884907+0000 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: created 2024-03-29T04:24:58.066690+0000 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: min_mon_release 19 (squid) 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: election_strategy: 1 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: 0: [v2:172.21.15.29:3300/0,v1:172.21.15.29:6789/0] mon.a 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: 1: [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon.b 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: 2: [v2:172.21.15.29:3301/0,v1:172.21.15.29:6790/0] mon.c 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: fsmap foo:1 {0=foo.smithi029.gsovzu=up:active} 1 up:standby 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: mgrmap e41: y(active, since 3m), standbys: x 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:42:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:42:17.372 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:17.372 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:17.372 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:17.372 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:16 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1045359964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:18.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:18 smithi029 ceph-mon[102350]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 479 op/s 2024-03-29T04:42:18.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:18 smithi029 ceph-mon[106624]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 479 op/s 2024-03-29T04:42:19.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:18 smithi099 ceph-mon[94036]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 479 op/s 2024-03-29T04:42:19.194 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:42:19.781 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:19.891 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:42:19.678+0000 7f94b7a0c700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-29T04:42:20.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:20.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:20.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:20.902 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_1' removed 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[102350]: pgmap v121: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 395 op/s 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1794911395' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[106624]: pgmap v121: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 395 op/s 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1794911395' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:20 smithi099 ceph-mon[94036]: pgmap v121: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 395 op/s 2024-03-29T04:42:21.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:20 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1794911395' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:42:21.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1794911395' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[102350]: failed to encode map e83 with expected crc 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[102350]: failed to encode map e83 with expected crc 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[102350]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1794911395' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:42:21.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[106624]: failed to encode map e83 with expected crc 2024-03-29T04:42:21.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[106624]: failed to encode map e83 with expected crc 2024-03-29T04:42:21.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[106624]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:42:21.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:21.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:22.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:21 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1794911395' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:42:22.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:21 smithi099 ceph-mon[94036]: failed to encode map e83 with expected crc 2024-03-29T04:42:22.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:21 smithi099 ceph-mon[94036]: failed to encode map e83 with expected crc 2024-03-29T04:42:22.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:21 smithi099 ceph-mon[94036]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:42:22.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:22.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:22.870 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-03-29T04:42:22.871 INFO:tasks.radosbench:Beginning radosbench... 2024-03-29T04:42:22.871 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-03-29T04:42:22.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool create unique_pool_2 16 2024-03-29T04:42:23.024 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:22 smithi029 ceph-mon[102350]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:23.024 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:22 smithi029 ceph-mon[106624]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:23.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:22 smithi099 ceph-mon[94036]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:23.501 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:23.704 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:42:23.704 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (5m) 17s ago 10m 23.5M - 0.25.0 c8568f914cd2 6fb7bc148154 2024-03-29T04:42:23.704 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi099 *:3000 error 3s ago 10m - - 2024-03-29T04:42:23.704 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.gsovzu smithi029 running (9m) 17s ago 9m 16.6M - 17.2.7-789-g4213e17c cf22eb8683be e6b24be4eed9 2024-03-29T04:42:23.704 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi099.jhvkhg smithi099 running (9m) 3s ago 9m 17.0M - 17.2.7-789-g4213e17c cf22eb8683be 2d2ff55bb03a 2024-03-29T04:42:23.704 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi099 *:8443,9283,8765 running (4m) 3s ago 15m 451M - 19.0.0-1959-g8da23624 54112f57da44 1c6cc4c8d8c3 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (5m) 17s ago 16m 501M - 19.0.0-1959-g8da23624 54112f57da44 280a22bdfead 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (4m) 17s ago 17m 47.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 4ef356385530 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi099 running (10s) 3s ago 15m 26.4M 2048M 19.0.0-1959-g8da23624 54112f57da44 a53b591d5ed7 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (119s) 17s ago 15m 34.0M 2048M 19.0.0-1959-g8da23624 54112f57da44 69712aa77fae 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (5m) 17s ago 10m 16.3M - 1.5.0 0da6a335fe13 9026c9798e07 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi099 *:9100 running (5m) 3s ago 10m 15.5M - 1.5.0 0da6a335fe13 52f151f28461 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (14m) 17s ago 14m 383M 2481M 17.2.7-789-g4213e17c cf22eb8683be 5e5c90bac829 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (14m) 17s ago 14m 336M 2481M 17.2.7-789-g4213e17c cf22eb8683be 6532f8ba40e9 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (13m) 17s ago 13m 468M 2481M 17.2.7-789-g4213e17c cf22eb8683be 817a836d7edf 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (13m) 17s ago 13m 488M 2481M 17.2.7-789-g4213e17c cf22eb8683be a4cb24ddb406 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi099 running (13m) 3s ago 13m 408M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1fab66997673 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi099 running (12m) 3s ago 12m 436M 2480M 17.2.7-789-g4213e17c cf22eb8683be 78420bf6aaa9 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi099 running (12m) 3s ago 12m 308M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1f39d06fae80 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi099 running (11m) 3s ago 11m 389M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6adf0209fbb4 2024-03-29T04:42:23.705 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi099 *:9095 running (4m) 3s ago 11m 62.1M - 2.43.0 a07b618ecd1d cfdb8be5d010 2024-03-29T04:42:24.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:23 smithi029 ceph-mon[102350]: failed to encode map e83 with expected crc 2024-03-29T04:42:24.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:23 smithi029 ceph-mon[106624]: failed to encode map e83 with expected crc 2024-03-29T04:42:24.197 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:23 smithi099 ceph-mon[94036]: failed to encode map e83 with expected crc 2024-03-29T04:42:24.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:42:25.102 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:24 smithi099 ceph-mon[94036]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:25.102 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:24 smithi099 ceph-mon[94036]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:25.189 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:42:24] "GET /metrics HTTP/1.1" 200 36142 "" "Prometheus/2.43.0" 2024-03-29T04:42:25.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:24 smithi029 ceph-mon[106624]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:25.189 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:24 smithi029 ceph-mon[106624]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:25.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:24 smithi029 ceph-mon[102350]: pgmap v124: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:25.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:24 smithi029 ceph-mon[102350]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:26.058 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:26.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:26.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[102350]: Reconfiguring daemon mon.a on smithi029 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:26.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:26.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:26 smithi029 ceph-mon[106624]: Reconfiguring daemon mon.a on smithi029 2024-03-29T04:42:26.850 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: pgmap v125: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: pgmap v126: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: pgmap v127: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:26.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:26 smithi099 ceph-mon[94036]: Reconfiguring daemon mon.a on smithi029 2024-03-29T04:42:26.924 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_2' created 2024-03-29T04:42:27.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:27 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2091630030' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-03-29T04:42:27.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:27 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-03-29T04:42:27.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:27 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-03-29T04:42:27.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:27 smithi099 ceph-mon[94036]: failed to encode map e84 with expected crc 2024-03-29T04:42:27.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:27 smithi099 ceph-mon[94036]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:42:27.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2091630030' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-03-29T04:42:27.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-03-29T04:42:27.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-03-29T04:42:27.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[102350]: failed to encode map e84 with expected crc 2024-03-29T04:42:27.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[102350]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:42:27.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2091630030' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-03-29T04:42:27.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-03-29T04:42:27.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-03-29T04:42:27.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[106624]: failed to encode map e84 with expected crc 2024-03-29T04:42:27.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:27 smithi029 ceph-mon[106624]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:42:28.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:28 smithi099 ceph-mon[94036]: failed to encode map e84 with expected crc 2024-03-29T04:42:28.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:28 smithi099 ceph-mon[94036]: pgmap v129: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:28.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:28 smithi099 ceph-mon[94036]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:42:28.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:28 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:28.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:28 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:28.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[106624]: failed to encode map e84 with expected crc 2024-03-29T04:42:28.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[106624]: pgmap v129: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[106624]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[102350]: failed to encode map e84 with expected crc 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[102350]: pgmap v129: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[102350]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:28.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:28 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:42:29.584 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[102350]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:42:29.585 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:29 smithi029 ceph-mon[106624]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:42:29.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: failed to encode map e85 with expected crc 2024-03-29T04:42:29.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:42:29.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:29 smithi099 ceph-mon[94036]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:42:30.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:30 smithi029 ceph-mon[102350]: failed to encode map e86 with expected crc 2024-03-29T04:42:30.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:30 smithi029 ceph-mon[102350]: failed to encode map e86 with expected crc 2024-03-29T04:42:30.796 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:30 smithi029 ceph-mon[102350]: pgmap v132: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:30.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:30 smithi029 ceph-mon[106624]: failed to encode map e86 with expected crc 2024-03-29T04:42:30.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:30 smithi029 ceph-mon[106624]: failed to encode map e86 with expected crc 2024-03-29T04:42:30.796 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:30 smithi029 ceph-mon[106624]: pgmap v132: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:30.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:30 smithi099 ceph-mon[94036]: failed to encode map e86 with expected crc 2024-03-29T04:42:30.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:30 smithi099 ceph-mon[94036]: failed to encode map e86 with expected crc 2024-03-29T04:42:30.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:30 smithi099 ceph-mon[94036]: pgmap v132: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:31.664 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:31.908 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: Reconfiguring daemon mgr.y on smithi029 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:32.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: Reconfiguring daemon mgr.y on smithi029 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:32.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:42:32.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:32 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.619 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.619 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:32.619 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: Reconfiguring daemon mgr.y on smithi029 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:42:32.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:32 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.702 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:42:34.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:34.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.786 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: Reconfiguring daemon mon.c on smithi029 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:42:34.787 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:34.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:34.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:34.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: Reconfiguring daemon mon.c on smithi029 2024-03-29T04:42:34.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.788 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:34.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: Reconfiguring daemon mon.c on smithi029 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:34.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:35.136 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:42:34] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.43.0" 2024-03-29T04:42:36.502 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-03-29T04:42:36.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:36 smithi029 ceph-mon[106624]: pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:36.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:36 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/288527906' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:42:36.689 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:36 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:42:36.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:36 smithi029 ceph-mon[102350]: pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:36.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:36 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/288527906' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:42:36.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:36 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:42:36.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:36 smithi099 ceph-mon[94036]: pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:36.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:36 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/288527906' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:42:36.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:36 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:42:37.325 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: failed to encode map e87 with expected crc 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: failed to encode map e87 with expected crc 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:42:37.612 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.0 on smithi029 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: failed to encode map e87 with expected crc 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: failed to encode map e87 with expected crc 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:37.613 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:37 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.0 on smithi029 2024-03-29T04:42:37.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:42:37.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: failed to encode map e87 with expected crc 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: failed to encode map e87 with expected crc 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:37.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:37 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.0 on smithi029 2024-03-29T04:42:38.348 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-03-29T04:42:38.351 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-03-29T04:42:38.352 INFO:teuthology.orchestra.run:waiting for 3000 2024-03-29T04:42:38.399 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-03-29T04:42:38.399 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-03-29T04:42:38.399 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_113431 2024-03-29T04:42:38.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:42:38.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-03-29T04:42:38.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:38 smithi029 ceph-mon[102350]: pgmap v137: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:38.749 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:38 smithi029 ceph-mon[102350]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:42:38.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:38 smithi029 ceph-mon[106624]: pgmap v137: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:38.750 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:38 smithi029 ceph-mon[106624]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:42:38.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:38 smithi099 ceph-mon[94036]: pgmap v137: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:38.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:38 smithi099 ceph-mon[94036]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:42:39.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 3840 3824 238.955 239 0.00448405 0.00416868 2024-03-29T04:42:40.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 7110 7094 221.657 204.375 0.00728409 0.00450059 2024-03-29T04:42:40.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[102350]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:40.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:40.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:40.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:42:40.689 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:40.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[106624]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:40.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:40.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:40.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:42:40.690 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:40 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:40.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:40 smithi099 ceph-mon[94036]: pgmap v138: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:40.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:40 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:40.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:40 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:40.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:40 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:42:40.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:40 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:41.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10169 10153 211.495 191.188 0.0029762 0.00472012 2024-03-29T04:42:41.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:41 smithi099 ceph-mon[94036]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:42:41.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:41 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.1 on smithi029 2024-03-29T04:42:41.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:41 smithi029 ceph-mon[102350]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:42:41.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:41 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.1 on smithi029 2024-03-29T04:42:41.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:41 smithi029 ceph-mon[106624]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:42:41.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:41 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.1 on smithi029 2024-03-29T04:42:42.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 13302 13286 207.57 195.812 0.00327581 0.00481005 2024-03-29T04:42:42.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:42 smithi099 ceph-mon[94036]: pgmap v139: 89 pgs: 89 active+clean; 404 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2024-03-29T04:42:42.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:42 smithi029 ceph-mon[102350]: pgmap v139: 89 pgs: 89 active+clean; 404 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2024-03-29T04:42:42.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:42 smithi029 ceph-mon[106624]: pgmap v139: 89 pgs: 89 active+clean; 404 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2024-03-29T04:42:43.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 15905 15889 198.591 162.688 0.00364013 0.00502736 2024-03-29T04:42:43.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:43.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:43.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:43 smithi099 ceph-mon[94036]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:42:43.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:42:43.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:43.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:43 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.2 on smithi029 2024-03-29T04:42:43.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:43.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:43.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[102350]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.2 on smithi029 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[106624]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:43.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:43 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.2 on smithi029 2024-03-29T04:42:44.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 17991 17975 187.22 130.375 0.00752969 0.00533473 2024-03-29T04:42:44.903 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:44 smithi029 ceph-mon[102350]: pgmap v140: 89 pgs: 89 active+clean; 404 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2024-03-29T04:42:44.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:44 smithi029 ceph-mon[106624]: pgmap v140: 89 pgs: 89 active+clean; 404 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2024-03-29T04:42:45.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:44 smithi099 ceph-mon[94036]: pgmap v140: 89 pgs: 89 active+clean; 404 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2024-03-29T04:42:45.189 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:42:44] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.43.0" 2024-03-29T04:42:45.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 20153 20137 179.776 135.125 0.0045168 0.00555631 2024-03-29T04:42:46.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 22266 22250 173.81 132.062 0.00365317 0.005747 2024-03-29T04:42:46.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:46 smithi099 ceph-mon[94036]: pgmap v141: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-03-29T04:42:46.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:46 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:46.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:46 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:46.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:46 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:42:46.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:46 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[102350]: pgmap v141: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[106624]: pgmap v141: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:42:46.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:46 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:47.400 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 24553 24537 170.377 142.938 0.00324858 0.00586384 2024-03-29T04:42:47.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:47 smithi029 ceph-mon[102350]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:42:47.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:47 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.3 on smithi029 2024-03-29T04:42:47.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:47 smithi029 ceph-mon[106624]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:42:47.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:47 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.3 on smithi029 2024-03-29T04:42:48.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:47 smithi099 ceph-mon[94036]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:42:48.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:47 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.3 on smithi029 2024-03-29T04:42:48.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 26959 26943 168.376 150.375 0.0146102 0.00593321 2024-03-29T04:42:48.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:48 smithi029 ceph-mon[102350]: pgmap v142: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-03-29T04:42:48.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:48 smithi029 ceph-mon[106624]: pgmap v142: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-03-29T04:42:49.119 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:48 smithi099 ceph-mon[94036]: pgmap v142: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-03-29T04:42:49.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 28475 28459 161.682 94.75 0.00347741 0.00617828 2024-03-29T04:42:50.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:50.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:50.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:50.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 16 29912 29896 155.692 89.8125 0.0141762 0.00641656 2024-03-29T04:42:50.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:50.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:50.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:50.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:50.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:50.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:50 smithi099 ceph-mon[94036]: pgmap v143: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-03-29T04:42:51.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:42:51.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:51.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 15 31907 31892 153.311 124.75 0.0155472 0.00651671 2024-03-29T04:42:51.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[106624]: pgmap v143: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-03-29T04:42:51.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[102350]: pgmap v143: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi029.gsovzu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:42:51.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:52.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:51 smithi099 ceph-mon[94036]: Reconfiguring mds.foo.smithi029.gsovzu (monmap changed)... 2024-03-29T04:42:52.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:51 smithi099 ceph-mon[94036]: Reconfiguring daemon mds.foo.smithi029.gsovzu on smithi029 2024-03-29T04:42:52.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 33895 33879 151.23 124.188 0.00273336 0.00657502 2024-03-29T04:42:52.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:51 smithi029 ceph-mon[102350]: Reconfiguring mds.foo.smithi029.gsovzu (monmap changed)... 2024-03-29T04:42:52.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:51 smithi029 ceph-mon[102350]: Reconfiguring daemon mds.foo.smithi029.gsovzu on smithi029 2024-03-29T04:42:52.439 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:51 smithi029 ceph-mon[106624]: Reconfiguring mds.foo.smithi029.gsovzu (monmap changed)... 2024-03-29T04:42:52.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:51 smithi029 ceph-mon[106624]: Reconfiguring daemon mds.foo.smithi029.gsovzu on smithi029 2024-03-29T04:42:53.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:52 smithi099 ceph-mon[94036]: pgmap v144: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-03-29T04:42:53.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 34873 34857 145.223 61.125 0.0113323 0.00688083 2024-03-29T04:42:53.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:52 smithi029 ceph-mon[102350]: pgmap v144: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-03-29T04:42:53.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:52 smithi029 ceph-mon[106624]: pgmap v144: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-03-29T04:42:54.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 36026 36010 140.65 72.0625 0.0077565 0.00710382 2024-03-29T04:42:55.189 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:54 smithi029 ceph-mon[102350]: pgmap v145: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-03-29T04:42:55.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:54 smithi029 ceph-mon[106624]: pgmap v145: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-03-29T04:42:55.190 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:42:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:42:54] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.43.0" 2024-03-29T04:42:55.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:54 smithi099 ceph-mon[94036]: pgmap v145: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-03-29T04:42:55.401 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 37813 37797 138.946 111.688 0.00740337 0.0071911 2024-03-29T04:42:56.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 39092 39076 135.667 79.9375 0.00423637 0.00734892 2024-03-29T04:42:57.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:56 smithi099 ceph-mon[94036]: pgmap v146: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-03-29T04:42:57.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 16 40353 40337 132.674 78.8125 0.00217965 0.0074504 2024-03-29T04:42:57.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:56 smithi029 ceph-mon[102350]: pgmap v146: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-03-29T04:42:57.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:56 smithi029 ceph-mon[106624]: pgmap v146: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2024-03-29T04:42:58.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:42:57 smithi099 ceph-mon[94036]: pgmap v147: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-03-29T04:42:58.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:42:58.405807+0000 min lat: 0.00144855 max lat: 0.403954 avg lat: 0.00769737 2024-03-29T04:42:58.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:42:58.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 41256 41240 128.862 56.4375 0.00228626 0.00769737 2024-03-29T04:42:58.439 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:42:57 smithi029 ceph-mon[102350]: pgmap v147: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-03-29T04:42:58.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:42:57 smithi029 ceph-mon[106624]: pgmap v147: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-03-29T04:42:59.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 42312 42296 125.868 66 0.00646466 0.00793943 2024-03-29T04:43:00.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 43117 43101 122.433 50.3125 0.00362051 0.00816256 2024-03-29T04:43:00.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:00 smithi099 ceph-mon[94036]: pgmap v148: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-03-29T04:43:00.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:00 smithi029 ceph-mon[102350]: pgmap v148: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-03-29T04:43:00.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:00 smithi029 ceph-mon[106624]: pgmap v148: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-03-29T04:43:01.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 15 44577 44562 121.08 91.3125 0.00698452 0.00825345 2024-03-29T04:43:02.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 46141 46125 120.105 97.6875 0.0235575 0.00831923 2024-03-29T04:43:02.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:02 smithi099 ceph-mon[94036]: pgmap v149: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-03-29T04:43:02.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:02 smithi029 ceph-mon[102350]: pgmap v149: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-03-29T04:43:02.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:02 smithi029 ceph-mon[106624]: pgmap v149: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 123 MiB/s wr, 1.96k op/s 2024-03-29T04:43:03.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 15 47470 47455 118.626 83.125 0.00572193 0.00842497 2024-03-29T04:43:04.402 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 48399 48383 116.294 58 0.00468607 0.0085666 2024-03-29T04:43:04.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:04 smithi099 ceph-mon[94036]: pgmap v150: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-03-29T04:43:04.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:04.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:04 smithi029 ceph-mon[102350]: pgmap v150: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-03-29T04:43:04.902 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:04.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:04 smithi029 ceph-mon[106624]: pgmap v150: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-03-29T04:43:04.903 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:05.189 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:43:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:43:04] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.43.0" 2024-03-29T04:43:05.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 50066 50050 115.845 104.188 0.0138918 0.00862587 2024-03-29T04:43:06.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 51742 51726 115.448 104.75 0.00902222 0.00865622 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: pgmap v151: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:06.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:06 smithi099 ceph-mon[94036]: Reconfiguring daemon mon.b on smithi099 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: pgmap v151: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[102350]: Reconfiguring daemon mon.b on smithi099 2024-03-29T04:43:06.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: pgmap v151: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:06.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:06 smithi029 ceph-mon[106624]: Reconfiguring daemon mon.b on smithi099 2024-03-29T04:43:07.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 53420 53404 115.083 104.875 0.00817826 0.00868426 2024-03-29T04:43:08.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 55034 55018 114.609 100.875 0.00382408 0.00871908 2024-03-29T04:43:08.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: pgmap v152: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-03-29T04:43:08.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:08.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:08.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:43:08.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:43:08.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:43:08.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:08.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:08 smithi099 ceph-mon[94036]: Reconfiguring daemon mgr.x on smithi099 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: pgmap v152: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[102350]: Reconfiguring daemon mgr.x on smithi099 2024-03-29T04:43:08.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: pgmap v152: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:08.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:08 smithi029 ceph-mon[106624]: Reconfiguring daemon mgr.x on smithi099 2024-03-29T04:43:09.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 56649 56633 114.168 100.938 0.00386036 0.00875345 2024-03-29T04:43:10.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 58196 58180 113.622 96.6875 0.0164418 0.00879528 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: pgmap v153: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:10.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:10 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.4 on smithi099 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: pgmap v153: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.4 on smithi099 2024-03-29T04:43:11.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: pgmap v153: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-03-29T04:43:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:43:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:43:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:10 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.4 on smithi099 2024-03-29T04:43:11.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 59765 59749 113.15 98.0625 0.00499816 0.00883195 2024-03-29T04:43:12.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 61295 61279 112.634 95.625 0.00937067 0.00887291 2024-03-29T04:43:13.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:12 smithi099 ceph-mon[94036]: pgmap v154: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-03-29T04:43:13.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:12 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:13.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:12 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:13.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:12 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:43:13.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:12 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[102350]: pgmap v154: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[106624]: pgmap v154: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-03-29T04:43:13.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:13.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:13.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:43:13.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:12 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:13.403 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 62691 62675 111.909 87.25 0.0104921 0.00892991 2024-03-29T04:43:14.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:13 smithi099 ceph-mon[94036]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:43:14.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:13 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.5 on smithi099 2024-03-29T04:43:14.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:13 smithi029 ceph-mon[102350]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:43:14.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:13 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.5 on smithi099 2024-03-29T04:43:14.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:13 smithi029 ceph-mon[106624]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:43:14.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:13 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.5 on smithi099 2024-03-29T04:43:14.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 64091 64075 111.23 87.5 0.00294835 0.00897896 2024-03-29T04:43:15.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:14 smithi099 ceph-mon[94036]: pgmap v155: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:43:15.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:14 smithi029 ceph-mon[102350]: pgmap v155: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:43:15.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:14 smithi029 ceph-mon[106624]: pgmap v155: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-03-29T04:43:15.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:43:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:43:14] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.43.0" 2024-03-29T04:43:15.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 65529 65513 110.653 89.875 0.00419618 0.00903184 2024-03-29T04:43:16.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 15 66973 66958 110.117 90.3125 0.0071076 0.00907449 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: pgmap v156: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:16.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:16 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.6 on smithi099 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: pgmap v156: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.6 on smithi099 2024-03-29T04:43:16.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:16.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: pgmap v156: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-03-29T04:43:16.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:16.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:43:16.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:43:16.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:16.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:16 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.6 on smithi099 2024-03-29T04:43:17.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 68384 68368 109.553 88.125 0.00799118 0.00912199 2024-03-29T04:43:18.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:43:18.407683+0000 min lat: 0.00144855 max lat: 0.403954 avg lat: 0.00917255 2024-03-29T04:43:18.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:43:18.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 69754 69738 108.955 85.625 0.0125946 0.00917255 2024-03-29T04:43:18.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:18 smithi099 ceph-mon[94036]: pgmap v157: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-03-29T04:43:18.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:18 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:18.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:18 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:18.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:18 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:43:18.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:18 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[102350]: pgmap v157: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[106624]: pgmap v157: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:18.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:18.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:43:18.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:18 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:19.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 71107 71091 108.36 84.5625 0.00878816 0.00922253 2024-03-29T04:43:19.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:19 smithi099 ceph-mon[94036]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:43:19.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:19 smithi099 ceph-mon[94036]: Reconfiguring daemon osd.7 on smithi099 2024-03-29T04:43:19.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:19.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:19 smithi029 ceph-mon[102350]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:43:19.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:19 smithi029 ceph-mon[102350]: Reconfiguring daemon osd.7 on smithi099 2024-03-29T04:43:19.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:19.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:19 smithi029 ceph-mon[106624]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:43:19.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:19 smithi029 ceph-mon[106624]: Reconfiguring daemon osd.7 on smithi099 2024-03-29T04:43:19.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:20.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 72381 72365 107.675 79.625 0.00685652 0.009272 2024-03-29T04:43:20.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:20 smithi099 ceph-mon[94036]: pgmap v158: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-03-29T04:43:20.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:20.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:20.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:43:20.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:20.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[106624]: pgmap v158: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-03-29T04:43:20.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:20.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:20.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:43:20.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[102350]: pgmap v158: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-03-29T04:43:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.jhvkhg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:43:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:21.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 73545 73529 106.863 72.75 0.0132572 0.00935307 2024-03-29T04:43:21.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:21 smithi099 ceph-mon[94036]: Reconfiguring mds.foo.smithi099.jhvkhg (monmap changed)... 2024-03-29T04:43:21.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:21 smithi099 ceph-mon[94036]: Reconfiguring daemon mds.foo.smithi099.jhvkhg on smithi099 2024-03-29T04:43:21.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:21 smithi029 ceph-mon[102350]: Reconfiguring mds.foo.smithi099.jhvkhg (monmap changed)... 2024-03-29T04:43:21.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:21 smithi029 ceph-mon[102350]: Reconfiguring daemon mds.foo.smithi099.jhvkhg on smithi099 2024-03-29T04:43:21.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:21 smithi029 ceph-mon[106624]: Reconfiguring mds.foo.smithi099.jhvkhg (monmap changed)... 2024-03-29T04:43:21.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:21 smithi029 ceph-mon[106624]: Reconfiguring daemon mds.foo.smithi099.jhvkhg on smithi099 2024-03-29T04:43:22.404 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 74866 74850 106.311 82.5625 0.010505 0.00940019 2024-03-29T04:43:22.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:22 smithi099 ceph-mon[94036]: pgmap v159: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-03-29T04:43:22.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:22 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:22.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:22 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:22.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:22 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:22.841 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[102350]: pgmap v159: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[106624]: pgmap v159: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:22.842 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:22 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:23.407 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 75905 75889 105.39 64.9375 0.00617829 0.0094822 2024-03-29T04:43:24.405 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 76842 76826 104.372 58.5625 0.00471755 0.0095743 2024-03-29T04:43:24.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:24 smithi099 ceph-mon[94036]: pgmap v160: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-03-29T04:43:24.830 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:24 smithi029 ceph-mon[102350]: pgmap v160: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-03-29T04:43:24.830 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:24 smithi029 ceph-mon[106624]: pgmap v160: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-03-29T04:43:25.190 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:43:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:43:24] "GET /metrics HTTP/1.1" 200 36332 "" "Prometheus/2.43.0" 2024-03-29T04:43:25.405 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 15 77958 77943 103.636 69.8125 0.0118977 0.00964359 2024-03-29T04:43:26.405 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 78858 78842 102.648 56.1875 0.00320041 0.00970564 2024-03-29T04:43:26.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:26 smithi099 ceph-mon[94036]: pgmap v161: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-03-29T04:43:26.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:26 smithi029 ceph-mon[106624]: pgmap v161: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-03-29T04:43:26.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:26 smithi029 ceph-mon[102350]: pgmap v161: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-03-29T04:43:27.406 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 79910 79894 101.895 65.75 0.0113545 0.00980886 2024-03-29T04:43:28.406 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 81011 80995 101.233 68.8125 0.010323 0.00985828 2024-03-29T04:43:29.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:28 smithi099 ceph-mon[94036]: pgmap v162: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-03-29T04:43:29.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:28 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:29.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:28 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:29.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:28 smithi029 ceph-mon[102350]: pgmap v162: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-03-29T04:43:29.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:28 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:29.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:28 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:29.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:28 smithi029 ceph-mon[106624]: pgmap v162: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-03-29T04:43:29.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:28 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:29.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:28 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:29.406 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 81922 81906 100.364 56.9375 0.00752095 0.00995831 2024-03-29T04:43:30.406 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 82762 82746 99.4435 52.5 0.00351524 0.0100218 2024-03-29T04:43:31.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:30 smithi099 ceph-mon[94036]: pgmap v163: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-03-29T04:43:31.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:30 smithi029 ceph-mon[102350]: pgmap v163: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-03-29T04:43:31.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:30 smithi029 ceph-mon[106624]: pgmap v163: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-03-29T04:43:31.406 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 83498 83482 98.4351 46 0.0153234 0.0101537 2024-03-29T04:43:32.406 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 84098 84082 97.3065 37.5 0.218839 0.0102207 2024-03-29T04:43:33.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:32 smithi099 ceph-mon[94036]: pgmap v164: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-03-29T04:43:33.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:32 smithi029 ceph-mon[102350]: pgmap v164: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-03-29T04:43:33.190 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:32 smithi029 ceph-mon[106624]: pgmap v164: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-03-29T04:43:33.407 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 84907 84891 96.4551 50.5625 0.0164485 0.0103622 2024-03-29T04:43:34.407 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 15 85859 85844 95.7962 59.5625 0.0158129 0.010433 2024-03-29T04:43:35.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:34 smithi099 ceph-mon[94036]: pgmap v165: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2024-03-29T04:43:35.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:35.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:34 smithi029 ceph-mon[102350]: pgmap v165: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2024-03-29T04:43:35.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:35.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:43:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:43:34] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.43.0" 2024-03-29T04:43:35.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:34 smithi029 ceph-mon[106624]: pgmap v165: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2024-03-29T04:43:35.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:35.407 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 86947 86931 95.3074 67.9375 0.0165227 0.0104864 2024-03-29T04:43:36.409 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 88058 88042 94.8579 69.4375 0.0123082 0.0105364 2024-03-29T04:43:36.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:36 smithi029 ceph-mon[102350]: pgmap v166: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:43:36.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:36 smithi029 ceph-mon[106624]: pgmap v166: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:43:37.121 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:36 smithi099 ceph-mon[94036]: pgmap v166: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-03-29T04:43:37.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 89116 89100 94.3708 66.125 0.031567 0.0105906 2024-03-29T04:43:38.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:38 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:38.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:38 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:38.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:38 smithi099 ceph-mon[94036]: pgmap v167: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-03-29T04:43:38.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:43:38.413078+0000 min lat: 0.00144855 max lat: 0.413958 avg lat: 0.0106478 2024-03-29T04:43:38.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:43:38.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 90141 90125 93.8656 64.0625 0.00915114 0.0106478 2024-03-29T04:43:38.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:38 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:38.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:38 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:38.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:38 smithi029 ceph-mon[102350]: pgmap v167: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-03-29T04:43:38.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:38 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:38.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:38 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:38.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:38 smithi029 ceph-mon[106624]: pgmap v167: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-03-29T04:43:39.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:39 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:39.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:39 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:39.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:39 smithi099 ceph-mon[94036]: pgmap v168: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-03-29T04:43:39.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:39 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:39.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 91281 91265 93.4948 71.25 0.0118974 0.0106903 2024-03-29T04:43:39.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:39.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:39.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[102350]: pgmap v168: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-03-29T04:43:39.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:39.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:39.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:39.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[106624]: pgmap v168: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2024-03-29T04:43:39.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:39 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:40.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 92399 92383 93.1137 69.875 0.0125227 0.0107346 2024-03-29T04:43:41.258 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:43:41.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:41 smithi099 ceph-mon[94036]: pgmap v169: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-03-29T04:43:41.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 93485 93469 92.7131 67.875 0.0211633 0.0107799 2024-03-29T04:43:41.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:41 smithi029 ceph-mon[102350]: pgmap v169: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-03-29T04:43:41.440 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:41 smithi029 ceph-mon[106624]: pgmap v169: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-03-29T04:43:41.768 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:43:42.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 94507 94491 92.2624 63.875 0.00959398 0.0108329 2024-03-29T04:43:43.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:43 smithi029 ceph-mon[102350]: pgmap v170: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-03-29T04:43:43.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:43 smithi029 ceph-mon[106624]: pgmap v170: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-03-29T04:43:43.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 95575 95559 91.8698 66.75 0.019297 0.0108801 2024-03-29T04:43:43.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:43 smithi099 ceph-mon[94036]: pgmap v170: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-03-29T04:43:44.360 INFO:teuthology.orchestra.run.smithi029.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:43:44.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 96534 96518 91.386 59.9375 0.0199353 0.0109358 2024-03-29T04:43:45.190 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:43:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:43:44] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.43.0" 2024-03-29T04:43:45.410 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 97262 97246 90.7011 45.5 0.00483164 0.0110187 2024-03-29T04:43:45.594 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: pgmap v171: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='client.54136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: pgmap v172: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: pgmap v173: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:45.595 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: Upgrade: Setting container_image for all mon 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: Upgrade: Setting container_image for all crash 2024-03-29T04:43:45.596 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[106624]: Upgrade: osd.0 is safe to restart 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: pgmap v171: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='client.54136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:45.597 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: pgmap v172: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: pgmap v173: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: Upgrade: Setting container_image for all mon 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:43:45.598 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:43:45.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: Upgrade: Setting container_image for all crash 2024-03-29T04:43:45.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:43:45.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:43:45.599 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:45 smithi029 ceph-mon[102350]: Upgrade: osd.0 is safe to restart 2024-03-29T04:43:45.620 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: pgmap v171: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='client.54136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: pgmap v172: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: pgmap v173: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:45.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: Upgrade: Setting container_image for all mon 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: Upgrade: Setting container_image for all crash 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:45.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:43:45.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:43:45.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:45 smithi099 ceph-mon[94036]: Upgrade: osd.0 is safe to restart 2024-03-29T04:43:45.642 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for half of osds to upgrade' 2024-03-29T04:43:46.411 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 98178 98162 90.2082 57.25 0.0102276 0.0110794 2024-03-29T04:43:46.813 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:43:47.411 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 99232 99216 89.8555 65.875 0.0286599 0.0111235 2024-03-29T04:43:47.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:47 smithi029 ceph-mon[102350]: pgmap v174: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:43:47.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:47 smithi029 ceph-mon[106624]: pgmap v174: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:43:47.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:47 smithi099 ceph-mon[94036]: pgmap v174: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-03-29T04:43:48.412 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 100132 100116 89.375 56.25 0.0283357 0.0111838 2024-03-29T04:43:48.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[102350]: Upgrade: Updating osd.0 2024-03-29T04:43:48.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:48.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:43:48.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:48.760 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[102350]: Deploying daemon osd.0 on smithi029 2024-03-29T04:43:48.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[106624]: Upgrade: Updating osd.0 2024-03-29T04:43:48.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:48.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:43:48.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:48.761 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:48 smithi029 ceph-mon[106624]: Deploying daemon osd.0 on smithi029 2024-03-29T04:43:48.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:48 smithi099 ceph-mon[94036]: Upgrade: Updating osd.0 2024-03-29T04:43:48.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:48 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:43:48.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:48 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:43:48.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:48 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:48.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:48 smithi099 ceph-mon[94036]: Deploying daemon osd.0 on smithi029 2024-03-29T04:43:49.412 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 101133 101117 88.9973 62.5625 0.0313713 0.01123 2024-03-29T04:43:49.690 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:49 smithi029 ceph-mon[102350]: pgmap v175: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-03-29T04:43:49.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:49.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:49 smithi029 ceph-mon[106624]: pgmap v175: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-03-29T04:43:49.691 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:49 smithi099 ceph-mon[94036]: pgmap v175: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-03-29T04:43:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:50.412 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 102142 102126 88.6371 63.0625 0.0160225 0.0112763 2024-03-29T04:43:50.752 INFO:teuthology.orchestra.run.smithi029.stdout:wait for half of osds to upgrade 2024-03-29T04:43:51.412 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 103127 103111 88.2661 61.5625 0.0147588 0.0113242 2024-03-29T04:43:51.845 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:51 smithi029 ceph-mon[102350]: pgmap v176: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-03-29T04:43:51.845 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:51 smithi029 ceph-mon[106624]: pgmap v176: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-03-29T04:43:51.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:51 smithi099 ceph-mon[94036]: pgmap v176: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2024-03-29T04:43:52.412 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 103937 103921 87.7574 50.625 0.0153068 0.0113899 2024-03-29T04:43:53.412 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 104722 104706 87.2415 49.0625 0.0230585 0.0114568 2024-03-29T04:43:53.548 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-03-29T04:43:53.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:53 smithi099 ceph-mon[94036]: pgmap v177: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-03-29T04:43:53.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:53 smithi029 ceph-mon[102350]: pgmap v177: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-03-29T04:43:53.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:53 smithi029 ceph-mon[106624]: pgmap v177: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-03-29T04:43:54.413 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 105518 105502 86.7469 49.75 0.0212798 0.0115217 2024-03-29T04:43:54.498 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:43:55.113 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:43:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:43:54] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.43.0" 2024-03-29T04:43:55.114 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:54 smithi029 systemd[1]: Stopping Ceph osd.0 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:43:55.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 106300 106284 86.255 48.875 0.0124649 0.0115881 2024-03-29T04:43:55.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:55 smithi099 ceph-mon[94036]: pgmap v178: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-03-29T04:43:55.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:55 smithi029 ceph-mon[102350]: pgmap v178: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-03-29T04:43:55.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:55 smithi029 ceph-mon[106624]: pgmap v178: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2024-03-29T04:43:56.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 107258 107242 85.9168 59.875 0.0130242 0.0116338 2024-03-29T04:43:56.837 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:56 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[46337]: 2024-03-29T04:43:56.700+0000 7f368bc5b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:43:56.837 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:56 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[46337]: 2024-03-29T04:43:56.700+0000 7f368bc5b700 -1 osd.0 87 *** Got signal Terminated *** 2024-03-29T04:43:56.837 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:56 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[46337]: 2024-03-29T04:43:56.700+0000 7f368bc5b700 -1 osd.0 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:43:57.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 108164 108148 85.546 56.625 0.0120089 0.0116841 2024-03-29T04:43:57.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:57 smithi099 ceph-mon[94036]: pgmap v179: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-03-29T04:43:57.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:57 smithi099 ceph-mon[94036]: osd.0 marked itself down and dead 2024-03-29T04:43:57.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:57 smithi029 ceph-mon[102350]: pgmap v179: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-03-29T04:43:57.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:57 smithi029 ceph-mon[102350]: osd.0 marked itself down and dead 2024-03-29T04:43:57.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:57 smithi029 ceph-mon[106624]: pgmap v179: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-03-29T04:43:57.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:57 smithi029 ceph-mon[106624]: osd.0 marked itself down and dead 2024-03-29T04:43:58.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:43:58.416893+0000 min lat: 0.00144855 max lat: 0.413958 avg lat: 0.0116933 2024-03-29T04:43:58.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:43:58.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 16 108311 108295 84.5916 9.1875 0.0047137 0.0116933 2024-03-29T04:43:58.440 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:58 smithi029 podman[115723]: 2024-03-29 04:43:58.16813635 +0000 UTC m=+2.381873712 container died 5e5c90bac829970385686eaa8812a5881ce689662618cf18274c7360e84d872f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, ceph=True) 2024-03-29T04:43:58.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:58 smithi099 ceph-mon[94036]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:43:58.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:58 smithi099 ceph-mon[94036]: failed to encode map e88 with expected crc 2024-03-29T04:43:58.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:58 smithi099 ceph-mon[94036]: osdmap e88: 8 total, 7 up, 8 in 2024-03-29T04:43:58.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:58 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4042485921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:43:58.875 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:58 smithi029 podman[115723]: 2024-03-29 04:43:58.603546596 +0000 UTC m=+2.817283943 container cleanup 5e5c90bac829970385686eaa8812a5881ce689662618cf18274c7360e84d872f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD) 2024-03-29T04:43:58.875 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:58 smithi029 bash[115723]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0 2024-03-29T04:43:58.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[102350]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:43:58.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[102350]: failed to encode map e88 with expected crc 2024-03-29T04:43:58.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[102350]: osdmap e88: 8 total, 7 up, 8 in 2024-03-29T04:43:58.875 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4042485921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:43:58.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[106624]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:43:58.875 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[106624]: failed to encode map e88 with expected crc 2024-03-29T04:43:58.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[106624]: osdmap e88: 8 total, 7 up, 8 in 2024-03-29T04:43:58.876 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:58 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4042485921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:43:59.190 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:58 smithi029 podman[115815]: 2024-03-29 04:43:58.877293293 +0000 UTC m=+0.701380736 container remove 5e5c90bac829970385686eaa8812a5881ce689662618cf18274c7360e84d872f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True) 2024-03-29T04:43:59.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 109345 109329 84.345 64.625 0.0103031 0.0118497 2024-03-29T04:43:59.833 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:59 smithi029 podman[115892]: 2024-03-29 04:43:59.516780815 +0000 UTC m=+0.035037018 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:43:59.833 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:43:59 smithi029 podman[115892]: 2024-03-29 04:43:59.651859107 +0000 UTC m=+0.170115284 container create 337ec0375719d78776cee49edd22e782fd0427d2354bbae787259c4fe5e89893 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD) 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e88 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: pgmap v181: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: osdmap e89: 8 total, 7 up, 8 in 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.834 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[102350]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e88 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: pgmap v181: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: osdmap e89: 8 total, 7 up, 8 in 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.835 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:43:59 smithi029 ceph-mon[106624]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e88 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: pgmap v181: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: osdmap e89: 8 total, 7 up, 8 in 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:43:59.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:43:59 smithi099 ceph-mon[94036]: failed to encode map e89 with expected crc 2024-03-29T04:44:00.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 110515 110499 84.2081 73.125 0.00485872 0.0118703 2024-03-29T04:44:00.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:00 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/962415166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:00.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:00 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/962415166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:00.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:00 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/962415166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:01.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 111564 111548 83.9834 65.5625 0.00496438 0.0119018 2024-03-29T04:44:01.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:01 smithi029 ceph-mon[102350]: pgmap v183: 89 pgs: 11 active+undersized+degraded, 14 active+undersized, 62 active+clean, 2 stale+active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 41278/326286 objects degraded (12.651%) 2024-03-29T04:44:01.768 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:01 smithi029 podman[115892]: 2024-03-29 04:44:01.510127563 +0000 UTC m=+2.028383755 container init 337ec0375719d78776cee49edd22e782fd0427d2354bbae787259c4fe5e89893 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, ceph=True) 2024-03-29T04:44:01.768 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:01 smithi029 podman[115892]: 2024-03-29 04:44:01.525555707 +0000 UTC m=+2.043811890 container start 337ec0375719d78776cee49edd22e782fd0427d2354bbae787259c4fe5e89893 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:44:01.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:01 smithi029 ceph-mon[106624]: pgmap v183: 89 pgs: 11 active+undersized+degraded, 14 active+undersized, 62 active+clean, 2 stale+active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 41278/326286 objects degraded (12.651%) 2024-03-29T04:44:01.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:01 smithi099 ceph-mon[94036]: pgmap v183: 89 pgs: 11 active+undersized+degraded, 14 active+undersized, 62 active+clean, 2 stale+active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 41278/326286 objects degraded (12.651%) 2024-03-29T04:44:01.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:01 smithi099 ceph-mon[94036]: Health check failed: Degraded data redundancy: 41278/326286 objects degraded (12.651%), 11 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:02.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:01 smithi029 ceph-mon[102350]: Health check failed: Degraded data redundancy: 41278/326286 objects degraded (12.651%), 11 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:02.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:01 smithi029 ceph-mon[106624]: Health check failed: Degraded data redundancy: 41278/326286 objects degraded (12.651%), 11 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:02.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 112576 112560 83.7365 63.25 0.0208295 0.0119366 2024-03-29T04:44:03.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 113491 113475 83.4242 57.1875 0.0140633 0.0119819 2024-03-29T04:44:03.870 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:03 smithi099 ceph-mon[94036]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 55160/331443 objects degraded (16.642%) 2024-03-29T04:44:03.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2312499905' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:03.940 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:03 smithi029 ceph-mon[102350]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 55160/331443 objects degraded (16.642%) 2024-03-29T04:44:03.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2312499905' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:03.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:03 smithi029 ceph-mon[106624]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 55160/331443 objects degraded (16.642%) 2024-03-29T04:44:03.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2312499905' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:04.344 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:03 smithi029 podman[115892]: 2024-03-29 04:44:03.955813166 +0000 UTC m=+4.474069346 container attach 337ec0375719d78776cee49edd22e782fd0427d2354bbae787259c4fe5e89893 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:44:04.344 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:03 smithi029 podman[115892]: 2024-03-29 04:44:03.965014378 +0000 UTC m=+4.483270557 container died 337ec0375719d78776cee49edd22e782fd0427d2354bbae787259c4fe5e89893 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:44:04.414 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 114330 114314 83.0638 52.4375 0.00388123 0.0120224 2024-03-29T04:44:05.190 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:05.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:05.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:44:04] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2024-03-29T04:44:05.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:05.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:05.370 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:05.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:05.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 115116 115100 82.6737 49.125 0.0267633 0.0120908 2024-03-29T04:44:06.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:05 smithi099 ceph-mon[94036]: pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 520 op/s; 55160/331443 objects degraded (16.642%) 2024-03-29T04:44:06.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:05 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2059101979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:06.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 115944 115928 82.3222 51.75 0.00387505 0.0121208 2024-03-29T04:44:06.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:05 smithi029 ceph-mon[102350]: pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 520 op/s; 55160/331443 objects degraded (16.642%) 2024-03-29T04:44:06.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:05 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2059101979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:06.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:05 smithi029 ceph-mon[106624]: pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 520 op/s; 55160/331443 objects degraded (16.642%) 2024-03-29T04:44:06.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:05 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2059101979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:07.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:06 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 55160/331443 objects degraded (16.642%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:07.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 116546 116530 81.82 37.625 0.00903419 0.0121803 2024-03-29T04:44:07.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:06 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 55160/331443 objects degraded (16.642%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:07.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:06 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 55160/331443 objects degraded (16.642%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:08.121 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:07 smithi099 ceph-mon[94036]: pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 57206/341943 objects degraded (16.730%) 2024-03-29T04:44:08.121 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:07 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1761377621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:08.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0067 2024-03-29T04:44:08.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 117591 2024-03-29T04:44:08.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-03-29T04:44:08.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-03-29T04:44:08.415 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 81.6544 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 38.4818 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 239 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 9.1875 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1306 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 615.708 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 3824 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 147 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0122423 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0221511 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 1.05383 2024-03-29T04:44:08.416 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.00144855 2024-03-29T04:44:08.420 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-03-29T04:44:08.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:07 smithi029 ceph-mon[102350]: pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 57206/341943 objects degraded (16.730%) 2024-03-29T04:44:08.440 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:07 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1761377621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:08.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:07 smithi029 ceph-mon[106624]: pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 57206/341943 objects degraded (16.730%) 2024-03-29T04:44:08.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:07 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1761377621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:10.088 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:09 smithi029 ceph-mon[106624]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 57487/345414 objects degraded (16.643%) 2024-03-29T04:44:10.088 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:09 smithi029 podman[115963]: 2024-03-29 04:44:09.746026189 +0000 UTC m=+7.844791660 container remove 337ec0375719d78776cee49edd22e782fd0427d2354bbae787259c4fe5e89893 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T04:44:10.088 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:09 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-29T04:44:10.088 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:09 smithi029 systemd[1]: Stopped Ceph osd.0 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:44:10.088 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:09 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.0.service: Consumed 14min 32.899s CPU time. 2024-03-29T04:44:10.089 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:09 smithi029 ceph-mon[102350]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 57487/345414 objects degraded (16.643%) 2024-03-29T04:44:10.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:09 smithi099 ceph-mon[94036]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 57487/345414 objects degraded (16.643%) 2024-03-29T04:44:10.441 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:10 smithi029 systemd[1]: Starting Ceph osd.0 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:44:10.820 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:10 smithi029 podman[116159]: 2024-03-29 04:44:10.349698613 +0000 UTC m=+0.031842326 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:44:10.821 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:10 smithi029 podman[116159]: 2024-03-29 04:44:10.491951069 +0000 UTC m=+0.174094820 container create 7ef8eed88fbfe580afee6a8a60e84790cd33f55829dcbcc49cc241338d35f2df (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-03-29T04:44:11.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:10 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2563074187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:11.191 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:10 smithi029 podman[116159]: 2024-03-29 04:44:10.823011185 +0000 UTC m=+0.505154908 container init 7ef8eed88fbfe580afee6a8a60e84790cd33f55829dcbcc49cc241338d35f2df (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-03-29T04:44:11.191 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:10 smithi029 podman[116159]: 2024-03-29 04:44:10.834130419 +0000 UTC m=+0.516274130 container start 7ef8eed88fbfe580afee6a8a60e84790cd33f55829dcbcc49cc241338d35f2df (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-29T04:44:11.191 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:10 smithi029 podman[116159]: 2024-03-29 04:44:10.948801323 +0000 UTC m=+0.630945030 container attach 7ef8eed88fbfe580afee6a8a60e84790cd33f55829dcbcc49cc241338d35f2df (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T04:44:11.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:10 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2563074187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:11.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:10 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2563074187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:11.941 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 bash[116159]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T04:44:11.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-29T04:44:11.943 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:44:11.943 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate[116172]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-29T04:44:11.943 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:11 smithi029 podman[116159]: 2024-03-29 04:44:11.625704025 +0000 UTC m=+1.307847739 container died 7ef8eed88fbfe580afee6a8a60e84790cd33f55829dcbcc49cc241338d35f2df (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-29T04:44:12.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:11 smithi099 ceph-mon[94036]: pgmap v188: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 86 B/s rd, 56 MiB/s wr, 971 op/s; 57964/347796 objects degraded (16.666%) 2024-03-29T04:44:12.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:11 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 57964/347796 objects degraded (16.666%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:12.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-mon[102350]: pgmap v188: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 86 B/s rd, 56 MiB/s wr, 971 op/s; 57964/347796 objects degraded (16.666%) 2024-03-29T04:44:12.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 57964/347796 objects degraded (16.666%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:12.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-mon[106624]: pgmap v188: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 86 B/s rd, 56 MiB/s wr, 971 op/s; 57964/347796 objects degraded (16.666%) 2024-03-29T04:44:12.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:11 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 57964/347796 objects degraded (16.666%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:13.349 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:13 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3218962000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:13.349 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:13 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3218962000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:13.350 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 podman[116312]: 2024-03-29 04:44:13.029315786 +0000 UTC m=+1.396255160 container remove 7ef8eed88fbfe580afee6a8a60e84790cd33f55829dcbcc49cc241338d35f2df (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-activate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T04:44:13.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:13 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3218962000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:13.677 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 podman[116396]: 2024-03-29 04:44:13.255488699 +0000 UTC m=+0.031514911 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:44:13.678 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 podman[116396]: 2024-03-29 04:44:13.389967002 +0000 UTC m=+0.165993204 container create 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5) 2024-03-29T04:44:13.941 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 podman[116396]: 2024-03-29 04:44:13.679912887 +0000 UTC m=+0.455939096 container init 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, RELEASE=squid-8da2362, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2) 2024-03-29T04:44:13.941 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 podman[116396]: 2024-03-29 04:44:13.697702934 +0000 UTC m=+0.473729142 container start 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-03-29T04:44:13.941 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 ceph-osd[116433]: -- 172.21.15.29:0/2948347881 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56041877d680 con 0x560418746c00 2024-03-29T04:44:13.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 bash[116396]: 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 2024-03-29T04:44:13.942 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 systemd[1]: Started Ceph osd.0 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:44:14.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:14 smithi099 ceph-mon[94036]: pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 901 op/s; 57909/346953 objects degraded (16.691%) 2024-03-29T04:44:14.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:14.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:14.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:14.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[102350]: pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 901 op/s; 57909/346953 objects degraded (16.691%) 2024-03-29T04:44:14.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:14.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:14.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:14.441 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[116410]: 2024-03-29T04:44:13.996+0000 7f0dc4e277c0 -1 Falling back to public interface 2024-03-29T04:44:14.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[106624]: pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 901 op/s; 57909/346953 objects degraded (16.691%) 2024-03-29T04:44:14.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:14.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:14.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:15.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:15 smithi029 ceph-mon[102350]: pgmap v190: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 758 op/s; 57909/346953 objects degraded (16.691%) 2024-03-29T04:44:15.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:44:14] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2024-03-29T04:44:15.191 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:15 smithi029 ceph-mon[106624]: pgmap v190: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 758 op/s; 57909/346953 objects degraded (16.691%) 2024-03-29T04:44:15.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:15 smithi099 ceph-mon[94036]: pgmap v190: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 758 op/s; 57909/346953 objects degraded (16.691%) 2024-03-29T04:44:16.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:16 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/76953153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:16.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:16 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/76953153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:16.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:16 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/76953153' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:17 smithi099 ceph-mon[94036]: pgmap v191: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.11k op/s; 55473/334197 objects degraded (16.599%) 2024-03-29T04:44:17.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:17 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 57909/346953 objects degraded (16.691%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:17.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:17 smithi029 ceph-mon[102350]: pgmap v191: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.11k op/s; 55473/334197 objects degraded (16.599%) 2024-03-29T04:44:17.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:17 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 57909/346953 objects degraded (16.691%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:17.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:17 smithi029 ceph-mon[106624]: pgmap v191: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.11k op/s; 55473/334197 objects degraded (16.599%) 2024-03-29T04:44:17.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:17 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 57909/346953 objects degraded (16.691%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:18.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:18 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2578591655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:18.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:18 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2578591655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:18.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:18 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2578591655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:19.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:19 smithi099 ceph-mon[94036]: pgmap v192: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 939 op/s; 55120/329922 objects degraded (16.707%) 2024-03-29T04:44:19.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:19.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:19 smithi029 ceph-mon[102350]: pgmap v192: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 939 op/s; 55120/329922 objects degraded (16.707%) 2024-03-29T04:44:19.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:19.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:19 smithi029 ceph-mon[106624]: pgmap v192: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 939 op/s; 55120/329922 objects degraded (16.707%) 2024-03-29T04:44:19.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:20.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:20.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:20.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:20.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:20.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:21.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:21 smithi099 ceph-mon[94036]: pgmap v193: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 1.06k op/s; 53513/322173 objects degraded (16.610%) 2024-03-29T04:44:21.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:21 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3185415533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:21.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:21.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:21.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[102350]: pgmap v193: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 1.06k op/s; 53513/322173 objects degraded (16.610%) 2024-03-29T04:44:21.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3185415533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:21.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:21.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:21.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[106624]: pgmap v193: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 1.06k op/s; 53513/322173 objects degraded (16.610%) 2024-03-29T04:44:21.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3185415533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:21.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:21.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:22.736 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:22 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 53513/322173 objects degraded (16.610%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:22.736 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:22 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 53513/322173 objects degraded (16.610%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:22.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:22 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 53513/322173 objects degraded (16.610%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:23.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[102350]: pgmap v194: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 3.9 MiB/s wr, 1.04k op/s; 52561/314625 objects degraded (16.706%) 2024-03-29T04:44:23.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2178754809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:23.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[102350]: from='osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:44:23.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[102350]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:44:23.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[106624]: pgmap v194: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 3.9 MiB/s wr, 1.04k op/s; 52561/314625 objects degraded (16.706%) 2024-03-29T04:44:23.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2178754809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:23.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[106624]: from='osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:44:23.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-mon[106624]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:44:23.692 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[116410]: 2024-03-29T04:44:23.434+0000 7f0dc4e277c0 -1 osd.0 87 log_to_monitors true 2024-03-29T04:44:23.692 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:44:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[116410]: 2024-03-29T04:44:23.551+0000 7f0db627f700 -1 osd.0 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:44:23.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:23 smithi099 ceph-mon[94036]: pgmap v194: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 3.9 MiB/s wr, 1.04k op/s; 52561/314625 objects degraded (16.706%) 2024-03-29T04:44:23.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:23 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2178754809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:23.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:23 smithi099 ceph-mon[94036]: from='osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:44:23.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:23 smithi099 ceph-mon[94036]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: from='osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: osdmap e90: 8 total, 7 up, 8 in 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:44:24.767 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[102350]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: from='osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: osdmap e90: 8 total, 7 up, 8 in 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:44:24.768 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-mon[106624]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:44:24.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: failed to encode map e90 with expected crc 2024-03-29T04:44:24.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: from='osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:44:24.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: osdmap e90: 8 total, 7 up, 8 in 2024-03-29T04:44:24.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:44:24.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:24 smithi099 ceph-mon[94036]: failed to encode map e90 with expected crc 2024-03-29T04:44:25.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:44:24] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: pgmap v196: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 52561/314625 objects degraded (16.706%) 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: failed to encode map e91 with expected crc 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522] boot 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: osdmap e91: 8 total, 8 up, 8 in 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: failed to encode map e91 with expected crc 2024-03-29T04:44:25.858 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: pgmap v196: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 52561/314625 objects degraded (16.706%) 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: failed to encode map e91 with expected crc 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522] boot 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: osdmap e91: 8 total, 8 up, 8 in 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: failed to encode map e91 with expected crc 2024-03-29T04:44:25.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:25 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: pgmap v196: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s; 52561/314625 objects degraded (16.706%) 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: failed to encode map e91 with expected crc 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: osd.0 [v2:172.21.15.29:6802/1167430522,v1:172.21.15.29:6803/1167430522] boot 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: osdmap e91: 8 total, 8 up, 8 in 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: failed to encode map e91 with expected crc 2024-03-29T04:44:25.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:25 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:44:26.526 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:44:26.433+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-03-29T04:44:26.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: osdmap e92: 8 total, 8 up, 8 in 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1695031057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:26.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:26 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 52561/314625 objects degraded (16.706%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:26.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: osdmap e92: 8 total, 8 up, 8 in 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1695031057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 52561/314625 objects degraded (16.706%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: osdmap e92: 8 total, 8 up, 8 in 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1695031057' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:26.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:26.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:44:26.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:26.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:26 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 52561/314625 objects degraded (16.706%), 15 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:27.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e92 with expected crc 2024-03-29T04:44:27.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: pgmap v199: 89 pgs: 14 peering, 8 active+undersized+degraded, 12 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 26279/307977 objects degraded (8.533%) 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: pgmap v200: 89 pgs: 14 peering, 8 active+undersized+degraded, 12 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 26279/307977 objects degraded (8.533%) 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:27 smithi099 ceph-mon[94036]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e92 with expected crc 2024-03-29T04:44:27.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: pgmap v199: 89 pgs: 14 peering, 8 active+undersized+degraded, 12 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 26279/307977 objects degraded (8.533%) 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: pgmap v200: 89 pgs: 14 peering, 8 active+undersized+degraded, 12 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 26279/307977 objects degraded (8.533%) 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[102350]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e92 with expected crc 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: pgmap v199: 89 pgs: 14 peering, 8 active+undersized+degraded, 12 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 26279/307977 objects degraded (8.533%) 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: pgmap v200: 89 pgs: 14 peering, 8 active+undersized+degraded, 12 active+undersized, 55 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 26279/307977 objects degraded (8.533%) 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:44:27.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:27.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:27 smithi029 ceph-mon[106624]: failed to encode map e93 with expected crc 2024-03-29T04:44:28.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:28 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4249872253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:28.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:28 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4249872253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:28.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:28 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4249872253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:29.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:29 smithi099 ceph-mon[94036]: pgmap v202: 89 pgs: 2 remapped+peering, 14 peering, 4 active+undersized+degraded, 9 active+undersized, 60 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 13116/305025 objects degraded (4.300%) 2024-03-29T04:44:29.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:29 smithi029 ceph-mon[102350]: pgmap v202: 89 pgs: 2 remapped+peering, 14 peering, 4 active+undersized+degraded, 9 active+undersized, 60 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 13116/305025 objects degraded (4.300%) 2024-03-29T04:44:29.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:29 smithi029 ceph-mon[106624]: pgmap v202: 89 pgs: 2 remapped+peering, 14 peering, 4 active+undersized+degraded, 9 active+undersized, 60 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 13116/305025 objects degraded (4.300%) 2024-03-29T04:44:31.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:31 smithi099 ceph-mon[94036]: pgmap v203: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 2 active+undersized+degraded, 78 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 17837/302199 objects degraded (5.902%); 11 KiB/s, 0 objects/s recovering 2024-03-29T04:44:31.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:31 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2241893620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:31.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:31 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 17837/302199 objects degraded (5.902%), 5 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:31.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:31 smithi029 ceph-mon[102350]: pgmap v203: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 2 active+undersized+degraded, 78 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 17837/302199 objects degraded (5.902%); 11 KiB/s, 0 objects/s recovering 2024-03-29T04:44:31.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:31 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2241893620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:31.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:31 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 17837/302199 objects degraded (5.902%), 5 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:31.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:31 smithi029 ceph-mon[106624]: pgmap v203: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 4 peering, 2 active+undersized+degraded, 78 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 17837/302199 objects degraded (5.902%); 11 KiB/s, 0 objects/s recovering 2024-03-29T04:44:31.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:31 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2241893620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:31.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:31 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 17837/302199 objects degraded (5.902%), 5 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:33.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:33 smithi099 ceph-mon[94036]: pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 11551/294297 objects degraded (3.925%); 1.5 MiB/s, 24 objects/s recovering 2024-03-29T04:44:33.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3739655546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:33.888 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:33 smithi029 ceph-mon[102350]: pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 11551/294297 objects degraded (3.925%); 1.5 MiB/s, 24 objects/s recovering 2024-03-29T04:44:33.889 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3739655546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:33.889 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:33 smithi029 ceph-mon[106624]: pgmap v204: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 11551/294297 objects degraded (3.925%); 1.5 MiB/s, 24 objects/s recovering 2024-03-29T04:44:33.889 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3739655546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:35.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:35.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:35.192 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:44:34] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-03-29T04:44:35.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:35.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:35.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:44:35.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:36.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:35 smithi099 ceph-mon[94036]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 11551/294297 objects degraded (3.925%); 1.3 MiB/s, 20 objects/s recovering 2024-03-29T04:44:36.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:35 smithi029 ceph-mon[102350]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 11551/294297 objects degraded (3.925%); 1.3 MiB/s, 20 objects/s recovering 2024-03-29T04:44:36.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:35 smithi029 ceph-mon[106624]: pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 11551/294297 objects degraded (3.925%); 1.3 MiB/s, 20 objects/s recovering 2024-03-29T04:44:37.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:36 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1205787440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:37.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:36 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11551/294297 objects degraded (3.925%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:37.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:36 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1205787440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:37.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:36 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11551/294297 objects degraded (3.925%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:37.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:36 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1205787440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:37.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:36 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11551/294297 objects degraded (3.925%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:38.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:37 smithi099 ceph-mon[94036]: pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 11551/286290 objects degraded (4.035%); 1.0 MiB/s, 16 objects/s recovering 2024-03-29T04:44:38.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:37 smithi029 ceph-mon[102350]: pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 11551/286290 objects degraded (4.035%); 1.0 MiB/s, 16 objects/s recovering 2024-03-29T04:44:38.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:37 smithi029 ceph-mon[106624]: pgmap v206: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 11551/286290 objects degraded (4.035%); 1.0 MiB/s, 16 objects/s recovering 2024-03-29T04:44:39.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:39 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2471841032' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:39.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:39 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2471841032' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:39.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:39 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2471841032' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:40.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:40 smithi099 ceph-mon[94036]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11398/282636 objects degraded (4.033%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:40.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:40 smithi029 ceph-mon[102350]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11398/282636 objects degraded (4.033%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:40.441 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:40 smithi029 ceph-mon[106624]: pgmap v207: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 11398/282636 objects degraded (4.033%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:41.941 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:41 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:44:41.445+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:42.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:42 smithi099 ceph-mon[94036]: pgmap v208: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 11398/278400 objects degraded (4.094%); 1.7 MiB/s, 26 objects/s recovering 2024-03-29T04:44:42.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:42 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3320987370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:42.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:42 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:42.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:42 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11398/278400 objects degraded (4.094%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:42.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[102350]: pgmap v208: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 11398/278400 objects degraded (4.094%); 1.7 MiB/s, 26 objects/s recovering 2024-03-29T04:44:42.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3320987370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:42.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:42.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11398/278400 objects degraded (4.094%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:42.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[106624]: pgmap v208: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 11398/278400 objects degraded (4.094%); 1.7 MiB/s, 26 objects/s recovering 2024-03-29T04:44:42.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3320987370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:42.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:42.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:42 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11398/278400 objects degraded (4.094%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:43.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:43 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:43.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:43 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:43.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:43 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:43.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:43 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:43.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:43 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:43.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:43 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:44.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:44 smithi099 ceph-mon[94036]: pgmap v209: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 11220/271548 objects degraded (4.132%); 2.6 MiB/s, 41 objects/s recovering 2024-03-29T04:44:44.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:44 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3671442131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:44.441 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:44 smithi029 ceph-mon[102350]: pgmap v209: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 11220/271548 objects degraded (4.132%); 2.6 MiB/s, 41 objects/s recovering 2024-03-29T04:44:44.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:44 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3671442131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:44.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:44 smithi029 ceph-mon[106624]: pgmap v209: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 11220/271548 objects degraded (4.132%); 2.6 MiB/s, 41 objects/s recovering 2024-03-29T04:44:44.442 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:44 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3671442131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:45.191 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:45 smithi029 ceph-mon[102350]: pgmap v210: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 11220/271548 objects degraded (4.132%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:45.192 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:44:44] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-03-29T04:44:45.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:45 smithi029 ceph-mon[106624]: pgmap v210: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 11220/271548 objects degraded (4.132%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:45.371 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:45 smithi099 ceph-mon[94036]: pgmap v210: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 11220/271548 objects degraded (4.132%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:47.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:47 smithi099 ceph-mon[94036]: pgmap v211: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 11220/263808 objects degraded (4.253%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:47.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:47 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11220/263808 objects degraded (4.253%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:47.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:47 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/279638701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:47.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:47 smithi029 ceph-mon[102350]: pgmap v211: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 11220/263808 objects degraded (4.253%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:47.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:47 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11220/263808 objects degraded (4.253%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:47.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:47 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/279638701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:47.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:47 smithi029 ceph-mon[106624]: pgmap v211: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 11220/263808 objects degraded (4.253%); 1.7 MiB/s, 27 objects/s recovering 2024-03-29T04:44:47.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:47 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11220/263808 objects degraded (4.253%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:47.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:47 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/279638701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:49.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:49 smithi099 ceph-mon[94036]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11106/260385 objects degraded (4.265%); 2.3 MiB/s, 46 objects/s recovering 2024-03-29T04:44:49.621 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:49.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:49 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2497065884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:49.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:49 smithi029 ceph-mon[102350]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11106/260385 objects degraded (4.265%); 2.3 MiB/s, 46 objects/s recovering 2024-03-29T04:44:49.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:49.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:49 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2497065884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:49.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:49 smithi029 ceph-mon[106624]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11106/260385 objects degraded (4.265%); 2.3 MiB/s, 46 objects/s recovering 2024-03-29T04:44:49.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:49.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:49 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2497065884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:51.622 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:51 smithi099 ceph-mon[94036]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 11106/256167 objects degraded (4.335%); 1.5 MiB/s, 33 objects/s recovering 2024-03-29T04:44:51.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:51 smithi029 ceph-mon[102350]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 11106/256167 objects degraded (4.335%); 1.5 MiB/s, 33 objects/s recovering 2024-03-29T04:44:51.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:51 smithi029 ceph-mon[106624]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 11106/256167 objects degraded (4.335%); 1.5 MiB/s, 33 objects/s recovering 2024-03-29T04:44:52.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:52 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11106/256167 objects degraded (4.335%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:52.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:52 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/137201489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:52.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:52 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11106/256167 objects degraded (4.335%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:52.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:52 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/137201489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:52.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:52 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11106/256167 objects degraded (4.335%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:52.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:52 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/137201489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:53.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:53 smithi029 ceph-mon[102350]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 11106/248907 objects degraded (4.462%); 1.5 MiB/s, 50 objects/s recovering 2024-03-29T04:44:53.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:53 smithi029 ceph-mon[106624]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 11106/248907 objects degraded (4.462%); 1.5 MiB/s, 50 objects/s recovering 2024-03-29T04:44:53.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:53 smithi099 ceph-mon[94036]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 11106/248907 objects degraded (4.462%); 1.5 MiB/s, 50 objects/s recovering 2024-03-29T04:44:55.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:44:54] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-03-29T04:44:55.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:55 smithi029 ceph-mon[102350]: pgmap v215: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11106/248907 objects degraded (4.462%); 608 KiB/s, 35 objects/s recovering 2024-03-29T04:44:55.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:55 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3357755656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:55.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:55 smithi029 ceph-mon[106624]: pgmap v215: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11106/248907 objects degraded (4.462%); 608 KiB/s, 35 objects/s recovering 2024-03-29T04:44:55.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:55 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3357755656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:55.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:55 smithi099 ceph-mon[94036]: pgmap v215: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11106/248907 objects degraded (4.462%); 608 KiB/s, 35 objects/s recovering 2024-03-29T04:44:55.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:55 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3357755656' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:56.941 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:44:56 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:44:56.462+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:57.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[102350]: pgmap v216: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 11106/241275 objects degraded (4.603%); 608 KiB/s, 35 objects/s recovering 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11106/241275 objects degraded (4.603%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4010260761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[106624]: pgmap v216: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 11106/241275 objects degraded (4.603%); 608 KiB/s, 35 objects/s recovering 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:57.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:57.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11106/241275 objects degraded (4.603%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:57.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:57 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4010260761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:57.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:57 smithi099 ceph-mon[94036]: pgmap v216: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s; 11106/241275 objects degraded (4.603%); 608 KiB/s, 35 objects/s recovering 2024-03-29T04:44:57.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:57 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:57.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:57 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:44:57.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:57 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:44:57.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:57 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11106/241275 objects degraded (4.603%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:44:57.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:57 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4010260761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:44:59.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:44:59 smithi029 ceph-mon[102350]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11106/237885 objects degraded (4.669%); 608 KiB/s, 51 objects/s recovering 2024-03-29T04:44:59.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:44:59 smithi029 ceph-mon[106624]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11106/237885 objects degraded (4.669%); 608 KiB/s, 51 objects/s recovering 2024-03-29T04:44:59.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:44:59 smithi099 ceph-mon[94036]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 11106/237885 objects degraded (4.669%); 608 KiB/s, 51 objects/s recovering 2024-03-29T04:45:00.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:00 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1592729840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:00.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:00 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1592729840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:00.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:00 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1592729840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:01.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:01 smithi029 ceph-mon[102350]: pgmap v218: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 11106/233796 objects degraded (4.750%); 0 B/s, 33 objects/s recovering 2024-03-29T04:45:01.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:01 smithi029 ceph-mon[106624]: pgmap v218: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 11106/233796 objects degraded (4.750%); 0 B/s, 33 objects/s recovering 2024-03-29T04:45:01.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:01 smithi099 ceph-mon[94036]: pgmap v218: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 11106/233796 objects degraded (4.750%); 0 B/s, 33 objects/s recovering 2024-03-29T04:45:02.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:02 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11106/233796 objects degraded (4.750%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:02.692 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:02 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/913270712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:02.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:02 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11106/233796 objects degraded (4.750%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:02.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:02 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/913270712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:02.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:02 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11106/233796 objects degraded (4.750%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:02.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:02 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/913270712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:03.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:03 smithi029 ceph-mon[102350]: pgmap v219: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 11106/227010 objects degraded (4.892%); 0 B/s, 59 objects/s recovering 2024-03-29T04:45:03.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:03 smithi029 ceph-mon[106624]: pgmap v219: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 11106/227010 objects degraded (4.892%); 0 B/s, 59 objects/s recovering 2024-03-29T04:45:03.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:03 smithi099 ceph-mon[94036]: pgmap v219: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 11106/227010 objects degraded (4.892%); 0 B/s, 59 objects/s recovering 2024-03-29T04:45:04.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:04.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:04.871 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:05.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:45:04] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2024-03-29T04:45:05.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:05 smithi029 ceph-mon[102350]: pgmap v220: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11106/227010 objects degraded (4.892%); 0 B/s, 42 objects/s recovering 2024-03-29T04:45:05.691 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:05 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/933987770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:05.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:05 smithi029 ceph-mon[106624]: pgmap v220: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11106/227010 objects degraded (4.892%); 0 B/s, 42 objects/s recovering 2024-03-29T04:45:05.692 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:05 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/933987770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:05.772 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:05 smithi099 ceph-mon[94036]: pgmap v220: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 11106/227010 objects degraded (4.892%); 0 B/s, 42 objects/s recovering 2024-03-29T04:45:05.772 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:05 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/933987770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: pgmap v221: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 11106/219318 objects degraded (5.064%); 0 B/s, 42 objects/s recovering 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11106/219318 objects degraded (5.064%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:07.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:07 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4290922370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: pgmap v221: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 11106/219318 objects degraded (5.064%); 0 B/s, 42 objects/s recovering 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11106/219318 objects degraded (5.064%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:07.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4290922370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: pgmap v221: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 11106/219318 objects degraded (5.064%); 0 B/s, 42 objects/s recovering 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: failed to encode map e94 with expected crc 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11106/219318 objects degraded (5.064%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:07.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:07 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4290922370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:08.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:08 smithi099 ceph-mon[94036]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:08 smithi099 ceph-mon[94036]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:08 smithi099 ceph-mon[94036]: osdmap e95: 8 total, 8 up, 8 in 2024-03-29T04:45:08.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:08 smithi099 ceph-mon[94036]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[102350]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[102350]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[102350]: osdmap e95: 8 total, 8 up, 8 in 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[102350]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[106624]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[106624]: failed to encode map e95 with expected crc 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[106624]: osdmap e95: 8 total, 8 up, 8 in 2024-03-29T04:45:08.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:08 smithi029 ceph-mon[106624]: failed to encode map e95 with expected crc 2024-03-29T04:45:09.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:09 smithi099 ceph-mon[94036]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 11106/216462 objects degraded (5.131%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:09.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:09 smithi029 ceph-mon[102350]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 11106/216462 objects degraded (5.131%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:09.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:09 smithi029 ceph-mon[106624]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 11106/216462 objects degraded (5.131%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:10.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:10 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4263631543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:10.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:10 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4263631543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:10.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:10 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4263631543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:11.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:11 smithi099 ceph-mon[94036]: pgmap v225: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 11106/212010 objects degraded (5.238%); 0 B/s, 24 objects/s recovering 2024-03-29T04:45:11.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:11.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:11 smithi029 ceph-mon[102350]: pgmap v225: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 11106/212010 objects degraded (5.238%); 0 B/s, 24 objects/s recovering 2024-03-29T04:45:11.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:11.942 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:11 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:45:11.483+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:11.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:11 smithi029 ceph-mon[106624]: pgmap v225: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 11106/212010 objects degraded (5.238%); 0 B/s, 24 objects/s recovering 2024-03-29T04:45:11.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:12.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:12 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:12.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:12 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:12.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:12 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11106/212010 objects degraded (5.238%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:12.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:12 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3968138127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11106/212010 objects degraded (5.238%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3968138127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11106/212010 objects degraded (5.238%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:12.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:12 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3968138127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:13.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:13 smithi099 ceph-mon[94036]: pgmap v226: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 9790/206541 objects degraded (4.740%); 2.3 MiB/s, 61 objects/s recovering 2024-03-29T04:45:13.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:13 smithi029 ceph-mon[102350]: pgmap v226: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 9790/206541 objects degraded (4.740%); 2.3 MiB/s, 61 objects/s recovering 2024-03-29T04:45:13.963 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:13 smithi029 ceph-mon[106624]: pgmap v226: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 9790/206541 objects degraded (4.740%); 2.3 MiB/s, 61 objects/s recovering 2024-03-29T04:45:15.191 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:45:14] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2024-03-29T04:45:15.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:15 smithi099 ceph-mon[94036]: pgmap v227: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 9790/205806 objects degraded (4.757%); 2.3 MiB/s, 61 objects/s recovering 2024-03-29T04:45:15.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3188287222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:15.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:15 smithi029 ceph-mon[102350]: pgmap v227: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 9790/205806 objects degraded (4.757%); 2.3 MiB/s, 61 objects/s recovering 2024-03-29T04:45:15.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3188287222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:15.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:15 smithi029 ceph-mon[106624]: pgmap v227: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 9790/205806 objects degraded (4.757%); 2.3 MiB/s, 61 objects/s recovering 2024-03-29T04:45:15.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3188287222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:17.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:17 smithi099 ceph-mon[94036]: pgmap v228: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9623/198039 objects degraded (4.859%); 2.9 MiB/s, 66 objects/s recovering 2024-03-29T04:45:17.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:17 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9623/198039 objects degraded (4.859%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:17.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:17 smithi029 ceph-mon[102350]: pgmap v228: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9623/198039 objects degraded (4.859%); 2.9 MiB/s, 66 objects/s recovering 2024-03-29T04:45:17.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:17 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9623/198039 objects degraded (4.859%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:17.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:17 smithi029 ceph-mon[106624]: pgmap v228: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9623/198039 objects degraded (4.859%); 2.9 MiB/s, 66 objects/s recovering 2024-03-29T04:45:17.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:17 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9623/198039 objects degraded (4.859%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:18.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:18 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2820741945' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:18.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:18 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2820741945' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:18.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:18 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2820741945' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:19.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:19 smithi099 ceph-mon[94036]: pgmap v229: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 9623/195276 objects degraded (4.928%); 2.6 MiB/s, 60 objects/s recovering 2024-03-29T04:45:19.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:19.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:19 smithi029 ceph-mon[102350]: pgmap v229: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 9623/195276 objects degraded (4.928%); 2.6 MiB/s, 60 objects/s recovering 2024-03-29T04:45:19.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:19.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:19 smithi029 ceph-mon[106624]: pgmap v229: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 9623/195276 objects degraded (4.928%); 2.6 MiB/s, 60 objects/s recovering 2024-03-29T04:45:19.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:20.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:20 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/457189984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:20.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:20 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/457189984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:20.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:20 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/457189984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:21.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:21 smithi099 ceph-mon[94036]: pgmap v230: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 9623/190449 objects degraded (5.053%); 2.4 MiB/s, 54 objects/s recovering 2024-03-29T04:45:21.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:21 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9623/190449 objects degraded (5.053%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:21.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:21 smithi029 ceph-mon[102350]: pgmap v230: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 9623/190449 objects degraded (5.053%); 2.4 MiB/s, 54 objects/s recovering 2024-03-29T04:45:21.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:21 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9623/190449 objects degraded (5.053%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:21.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:21 smithi029 ceph-mon[106624]: pgmap v230: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 9623/190449 objects degraded (5.053%); 2.4 MiB/s, 54 objects/s recovering 2024-03-29T04:45:21.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:21 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9623/190449 objects degraded (5.053%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:23.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:23 smithi099 ceph-mon[94036]: pgmap v231: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9471/183990 objects degraded (5.148%); 3.2 MiB/s, 52 objects/s recovering 2024-03-29T04:45:23.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:23 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3050427891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:23.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:23 smithi029 ceph-mon[102350]: pgmap v231: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9471/183990 objects degraded (5.148%); 3.2 MiB/s, 52 objects/s recovering 2024-03-29T04:45:23.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:23 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3050427891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:23.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:23 smithi029 ceph-mon[106624]: pgmap v231: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 9471/183990 objects degraded (5.148%); 3.2 MiB/s, 52 objects/s recovering 2024-03-29T04:45:23.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:23 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3050427891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:25.192 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:45:24] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2024-03-29T04:45:25.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:25 smithi099 ceph-mon[94036]: pgmap v232: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 9471/183321 objects degraded (5.166%); 1.7 MiB/s, 28 objects/s recovering 2024-03-29T04:45:25.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:25 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4236452678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:25.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:25 smithi029 ceph-mon[102350]: pgmap v232: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 9471/183321 objects degraded (5.166%); 1.7 MiB/s, 28 objects/s recovering 2024-03-29T04:45:25.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4236452678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:25 smithi029 ceph-mon[106624]: pgmap v232: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 9471/183321 objects degraded (5.166%); 1.7 MiB/s, 28 objects/s recovering 2024-03-29T04:45:25.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:25 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4236452678' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:26.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:26 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9471/183321 objects degraded (5.166%), 6 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:26.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:26 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9471/183321 objects degraded (5.166%), 6 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:26.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:26.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:26 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9471/183321 objects degraded (5.166%), 6 pgs degraded (PG_DEGRADED) 2024-03-29T04:45:27.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:27 smithi099 ceph-mon[94036]: pgmap v233: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 9471/176463 objects degraded (5.367%); 1.7 MiB/s, 41 objects/s recovering 2024-03-29T04:45:27.895 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:27 smithi029 ceph-mon[102350]: pgmap v233: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 9471/176463 objects degraded (5.367%); 1.7 MiB/s, 41 objects/s recovering 2024-03-29T04:45:27.896 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:27 smithi029 ceph-mon[106624]: pgmap v233: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 9471/176463 objects degraded (5.367%); 1.7 MiB/s, 41 objects/s recovering 2024-03-29T04:45:28.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:28 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1430945554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:28.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:28 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1430945554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:28.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:28 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1430945554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:29.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:29 smithi099 ceph-mon[94036]: pgmap v234: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9471/173889 objects degraded (5.447%); 810 KiB/s, 27 objects/s recovering 2024-03-29T04:45:29.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:29 smithi029 ceph-mon[106624]: pgmap v234: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9471/173889 objects degraded (5.447%); 810 KiB/s, 27 objects/s recovering 2024-03-29T04:45:29.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:29 smithi029 ceph-mon[102350]: pgmap v234: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9471/173889 objects degraded (5.447%); 810 KiB/s, 27 objects/s recovering 2024-03-29T04:45:31.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:31 smithi099 ceph-mon[94036]: pgmap v235: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9471/169344 objects degraded (5.593%); 810 KiB/s, 27 objects/s recovering 2024-03-29T04:45:31.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:31 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1310302718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:31.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:31 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:31.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:31 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:31.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:31 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9471/169344 objects degraded (5.593%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:31.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[102350]: pgmap v235: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9471/169344 objects degraded (5.593%); 810 KiB/s, 27 objects/s recovering 2024-03-29T04:45:31.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1310302718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9471/169344 objects degraded (5.593%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[106624]: pgmap v235: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9471/169344 objects degraded (5.593%); 810 KiB/s, 27 objects/s recovering 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1310302718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:31.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:31 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9471/169344 objects degraded (5.593%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:33.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:33 smithi099 ceph-mon[94036]: pgmap v236: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 9471/163422 objects degraded (5.795%); 810 KiB/s, 42 objects/s recovering 2024-03-29T04:45:33.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1625482224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:33.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:33 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:33.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:33 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:33.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[102350]: pgmap v236: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 9471/163422 objects degraded (5.795%); 810 KiB/s, 42 objects/s recovering 2024-03-29T04:45:33.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1625482224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:33.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:33.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:33.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[106624]: pgmap v236: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 9471/163422 objects degraded (5.795%); 810 KiB/s, 42 objects/s recovering 2024-03-29T04:45:33.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1625482224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:33.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:33.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:33 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:34.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:34.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:34.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:34.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:34.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:34.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:34.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:34.897 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:34.897 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:45:34.693+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:34.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:34.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:34.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:34.898 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:45:35.192 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:45:34] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2024-03-29T04:45:35.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9471/162828 objects degraded (5.817%); 0 B/s, 28 objects/s recovering 2024-03-29T04:45:35.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: pgmap v238: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 9471/162828 objects degraded (5.817%); 0 B/s, 33 objects/s recovering 2024-03-29T04:45:35.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:35.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:35.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:35.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:35 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:35.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9471/162828 objects degraded (5.817%); 0 B/s, 28 objects/s recovering 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: pgmap v238: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 9471/162828 objects degraded (5.817%); 0 B/s, 33 objects/s recovering 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:35.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: pgmap v237: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9471/162828 objects degraded (5.817%); 0 B/s, 28 objects/s recovering 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: pgmap v238: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 9471/162828 objects degraded (5.817%); 0 B/s, 33 objects/s recovering 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:35.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:35 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:36.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:36 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2010949116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:36.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:36 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9471/162828 objects degraded (5.817%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:36.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:36 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2010949116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:36.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:36 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9471/162828 objects degraded (5.817%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:37.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:36 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2010949116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:37.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:36 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9471/162828 objects degraded (5.817%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:37.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:37 smithi029 ceph-mon[102350]: pgmap v239: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 9471/155529 objects degraded (6.090%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:37.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:37 smithi029 ceph-mon[106624]: pgmap v239: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 9471/155529 objects degraded (6.090%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:38.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:37 smithi099 ceph-mon[94036]: pgmap v239: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 9471/155529 objects degraded (6.090%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:38.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1309119137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:38.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1309119137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:39.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1309119137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:39.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:39 smithi029 ceph-mon[102350]: pgmap v240: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9471/152874 objects degraded (6.195%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:39.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:39 smithi029 ceph-mon[106624]: pgmap v240: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9471/152874 objects degraded (6.195%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:40.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:39 smithi099 ceph-mon[94036]: pgmap v240: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9471/152874 objects degraded (6.195%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:41.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:41 smithi029 ceph-mon[102350]: pgmap v241: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9471/147924 objects degraded (6.403%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:41.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:41 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2536389263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:41.943 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:41 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9471/147924 objects degraded (6.403%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:41.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:41 smithi029 ceph-mon[106624]: pgmap v241: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9471/147924 objects degraded (6.403%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:41.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:41 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2536389263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:41.944 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:41 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9471/147924 objects degraded (6.403%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:42.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:41 smithi099 ceph-mon[94036]: pgmap v241: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9471/147924 objects degraded (6.403%); 0 B/s, 36 objects/s recovering 2024-03-29T04:45:42.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:41 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2536389263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:42.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:41 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9471/147924 objects degraded (6.403%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:43.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:43 smithi029 ceph-mon[102350]: pgmap v242: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9471/141549 objects degraded (6.691%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:43.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:43 smithi029 ceph-mon[106624]: pgmap v242: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9471/141549 objects degraded (6.691%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:44.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:43 smithi099 ceph-mon[94036]: pgmap v242: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9471/141549 objects degraded (6.691%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:44.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:44 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3831090105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:44.943 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:45:44] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2024-03-29T04:45:44.943 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:44 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3831090105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:45.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:44 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3831090105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:45.942 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:45 smithi029 ceph-mon[102350]: pgmap v243: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9471/140805 objects degraded (6.726%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:45.942 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:45 smithi029 ceph-mon[106624]: pgmap v243: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9471/140805 objects degraded (6.726%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:46.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:45 smithi099 ceph-mon[94036]: pgmap v243: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9471/140805 objects degraded (6.726%); 0 B/s, 39 objects/s recovering 2024-03-29T04:45:47.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:46 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1917788604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:47.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:46 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9471/140805 objects degraded (6.726%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:47.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:46 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1917788604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:47.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:46 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9471/140805 objects degraded (6.726%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:47.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:46 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1917788604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:47.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:46 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9471/140805 objects degraded (6.726%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:48.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: pgmap v244: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 9471/133143 objects degraded (7.113%); 0 B/s, 51 objects/s recovering 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: osdmap e96: 8 total, 8 up, 8 in 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:47 smithi099 ceph-mon[94036]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: pgmap v244: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 9471/133143 objects degraded (7.113%); 0 B/s, 51 objects/s recovering 2024-03-29T04:45:48.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: osdmap e96: 8 total, 8 up, 8 in 2024-03-29T04:45:48.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[102350]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: pgmap v244: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 9471/133143 objects degraded (7.113%); 0 B/s, 51 objects/s recovering 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: osdmap e96: 8 total, 8 up, 8 in 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:48.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:47 smithi029 ceph-mon[106624]: failed to encode map e96 with expected crc 2024-03-29T04:45:49.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:48 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:48 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:48 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:48 smithi099 ceph-mon[94036]: osdmap e97: 8 total, 8 up, 8 in 2024-03-29T04:45:49.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:48 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:48 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[102350]: osdmap e97: 8 total, 8 up, 8 in 2024-03-29T04:45:49.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[106624]: osdmap e97: 8 total, 8 up, 8 in 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:48 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:49 smithi099 ceph-mon[94036]: pgmap v247: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8471/130431 objects degraded (6.495%); 64 KiB/s, 54 objects/s recovering 2024-03-29T04:45:50.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:49 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:49 smithi099 ceph-mon[94036]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:49 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/798917409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:50.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:50.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:50.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[102350]: pgmap v247: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8471/130431 objects degraded (6.495%); 64 KiB/s, 54 objects/s recovering 2024-03-29T04:45:50.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[102350]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/798917409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:45:49.711+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[106624]: pgmap v247: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8471/130431 objects degraded (6.495%); 64 KiB/s, 54 objects/s recovering 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[106624]: failed to encode map e97 with expected crc 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/798917409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:50.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:51.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:50 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:51.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:50 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:51.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:50 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:51.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:50 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:51.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:50 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:45:51.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:50 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:45:52.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:51 smithi099 ceph-mon[94036]: pgmap v248: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 8471/127011 objects degraded (6.670%); 64 KiB/s, 28 objects/s recovering 2024-03-29T04:45:52.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:51 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2233112333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:52.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:51 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 8471/127011 objects degraded (6.670%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:52.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:51 smithi029 ceph-mon[102350]: pgmap v248: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 8471/127011 objects degraded (6.670%); 64 KiB/s, 28 objects/s recovering 2024-03-29T04:45:52.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:51 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2233112333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:52.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:51 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 8471/127011 objects degraded (6.670%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:52.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:51 smithi029 ceph-mon[106624]: pgmap v248: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 8471/127011 objects degraded (6.670%); 64 KiB/s, 28 objects/s recovering 2024-03-29T04:45:52.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:51 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2233112333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:52.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:51 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 8471/127011 objects degraded (6.670%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:54.019 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:53 smithi029 ceph-mon[102350]: pgmap v249: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 7971/122667 objects degraded (6.498%); 4.0 MiB/s, 96 objects/s recovering 2024-03-29T04:45:54.019 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:53 smithi029 ceph-mon[106624]: pgmap v249: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 7971/122667 objects degraded (6.498%); 4.0 MiB/s, 96 objects/s recovering 2024-03-29T04:45:54.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:53 smithi099 ceph-mon[94036]: pgmap v249: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 7971/122667 objects degraded (6.498%); 4.0 MiB/s, 96 objects/s recovering 2024-03-29T04:45:55.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:54 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2152653790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:55.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:54 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2152653790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:55.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:45:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:45:54] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2024-03-29T04:45:55.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:54 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2152653790' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:56.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:55 smithi099 ceph-mon[94036]: pgmap v250: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 7971/122043 objects degraded (6.531%); 4.0 MiB/s, 68 objects/s recovering 2024-03-29T04:45:56.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:55 smithi029 ceph-mon[102350]: pgmap v250: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 7971/122043 objects degraded (6.531%); 4.0 MiB/s, 68 objects/s recovering 2024-03-29T04:45:56.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:55 smithi029 ceph-mon[106624]: pgmap v250: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 7971/122043 objects degraded (6.531%); 4.0 MiB/s, 68 objects/s recovering 2024-03-29T04:45:57.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:56 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 7971/122043 objects degraded (6.531%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:57.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:56 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 7971/122043 objects degraded (6.531%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:57.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:56 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 7971/122043 objects degraded (6.531%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:45:58.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:57 smithi099 ceph-mon[94036]: pgmap v251: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 7971/115053 objects degraded (6.928%); 3.2 MiB/s, 56 objects/s recovering 2024-03-29T04:45:58.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:57 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2608326903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:58.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:57 smithi029 ceph-mon[102350]: pgmap v251: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 7971/115053 objects degraded (6.928%); 3.2 MiB/s, 56 objects/s recovering 2024-03-29T04:45:58.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:57 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2608326903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:45:58.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:57 smithi029 ceph-mon[106624]: pgmap v251: 89 pgs: 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 7971/115053 objects degraded (6.928%); 3.2 MiB/s, 56 objects/s recovering 2024-03-29T04:45:58.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:57 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2608326903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:00.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:59 smithi099 ceph-mon[94036]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 7910/112662 objects degraded (7.021%); 3.2 MiB/s, 66 objects/s recovering 2024-03-29T04:46:00.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:45:59 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4067778242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:00.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:59 smithi029 ceph-mon[102350]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 7910/112662 objects degraded (7.021%); 3.2 MiB/s, 66 objects/s recovering 2024-03-29T04:46:00.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:45:59 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4067778242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:00.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:59 smithi029 ceph-mon[106624]: pgmap v252: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 7910/112662 objects degraded (7.021%); 3.2 MiB/s, 66 objects/s recovering 2024-03-29T04:46:00.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:45:59 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4067778242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:02.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:01 smithi099 ceph-mon[94036]: pgmap v253: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 7910/108048 objects degraded (7.321%); 2.9 MiB/s, 60 objects/s recovering 2024-03-29T04:46:02.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:01 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 7910/108048 objects degraded (7.321%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:02.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:01 smithi029 ceph-mon[102350]: pgmap v253: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 7910/108048 objects degraded (7.321%); 2.9 MiB/s, 60 objects/s recovering 2024-03-29T04:46:02.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:01 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 7910/108048 objects degraded (7.321%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:02.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:01 smithi029 ceph-mon[106624]: pgmap v253: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 7910/108048 objects degraded (7.321%); 2.9 MiB/s, 60 objects/s recovering 2024-03-29T04:46:02.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:01 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 7910/108048 objects degraded (7.321%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:03.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:02 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3409116624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:03.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:02 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3409116624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:03.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:02 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3409116624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:04.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:03 smithi099 ceph-mon[94036]: pgmap v254: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 7910/101670 objects degraded (7.780%); 2.9 MiB/s, 79 objects/s recovering 2024-03-29T04:46:04.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:03 smithi029 ceph-mon[102350]: pgmap v254: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 7910/101670 objects degraded (7.780%); 2.9 MiB/s, 79 objects/s recovering 2024-03-29T04:46:04.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:03 smithi029 ceph-mon[106624]: pgmap v254: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 7910/101670 objects degraded (7.780%); 2.9 MiB/s, 79 objects/s recovering 2024-03-29T04:46:05.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:05.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:04 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4248144118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:05.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:05.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:05.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4248144118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:05.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:05.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:05.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4248144118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:05.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:05.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:46:04.729+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:05.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:46:04] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2024-03-29T04:46:06.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:05 smithi099 ceph-mon[94036]: pgmap v255: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7910/100932 objects degraded (7.837%); 325 KiB/s, 33 objects/s recovering 2024-03-29T04:46:06.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:05 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:06.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:05 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:05 smithi029 ceph-mon[102350]: pgmap v255: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7910/100932 objects degraded (7.837%); 325 KiB/s, 33 objects/s recovering 2024-03-29T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:05 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:06.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:05 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:05 smithi029 ceph-mon[106624]: pgmap v255: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7910/100932 objects degraded (7.837%); 325 KiB/s, 33 objects/s recovering 2024-03-29T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:05 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:06.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:05 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:07.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:06 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 7910/100932 objects degraded (7.837%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:07.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:06 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 7910/100932 objects degraded (7.837%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:07.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:06 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 7910/100932 objects degraded (7.837%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:08.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:07 smithi099 ceph-mon[94036]: pgmap v256: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7910/93087 objects degraded (8.497%); 325 KiB/s, 33 objects/s recovering 2024-03-29T04:46:08.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:07 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3329170117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:08.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:07 smithi029 ceph-mon[102350]: pgmap v256: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7910/93087 objects degraded (8.497%); 325 KiB/s, 33 objects/s recovering 2024-03-29T04:46:08.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:07 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3329170117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:08.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:07 smithi029 ceph-mon[106624]: pgmap v256: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7910/93087 objects degraded (8.497%); 325 KiB/s, 33 objects/s recovering 2024-03-29T04:46:08.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:07 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3329170117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:10.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:09 smithi099 ceph-mon[94036]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 7910/90264 objects degraded (8.763%); 325 KiB/s, 53 objects/s recovering 2024-03-29T04:46:10.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:09 smithi029 ceph-mon[102350]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 7910/90264 objects degraded (8.763%); 325 KiB/s, 53 objects/s recovering 2024-03-29T04:46:10.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:09 smithi029 ceph-mon[106624]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 7910/90264 objects degraded (8.763%); 325 KiB/s, 53 objects/s recovering 2024-03-29T04:46:11.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:10 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2911237293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:11.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:10 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2911237293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:11.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:10 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2911237293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:12.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:11 smithi099 ceph-mon[94036]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 7910/85305 objects degraded (9.273%); 0 B/s, 38 objects/s recovering 2024-03-29T04:46:12.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:11 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 7910/85305 objects degraded (9.273%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:12.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:11 smithi029 ceph-mon[102350]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 7910/85305 objects degraded (9.273%); 0 B/s, 38 objects/s recovering 2024-03-29T04:46:12.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:11 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 7910/85305 objects degraded (9.273%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:12.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:11 smithi029 ceph-mon[106624]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 7910/85305 objects degraded (9.273%); 0 B/s, 38 objects/s recovering 2024-03-29T04:46:12.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:11 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 7910/85305 objects degraded (9.273%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:13.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:12 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1834022433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:13.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:12 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1834022433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:13.192 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:12 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1834022433' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:14.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:13 smithi099 ceph-mon[94036]: pgmap v259: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 7910/78945 objects degraded (10.020%); 0 B/s, 58 objects/s recovering 2024-03-29T04:46:14.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:13 smithi029 ceph-mon[102350]: pgmap v259: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 7910/78945 objects degraded (10.020%); 0 B/s, 58 objects/s recovering 2024-03-29T04:46:14.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:13 smithi029 ceph-mon[106624]: pgmap v259: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 7910/78945 objects degraded (10.020%); 0 B/s, 58 objects/s recovering 2024-03-29T04:46:15.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:46:14] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2024-03-29T04:46:16.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:15 smithi099 ceph-mon[94036]: pgmap v260: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 7910/78192 objects degraded (10.116%); 0 B/s, 39 objects/s recovering 2024-03-29T04:46:16.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2151731289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:16.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:15 smithi029 ceph-mon[102350]: pgmap v260: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 7910/78192 objects degraded (10.116%); 0 B/s, 39 objects/s recovering 2024-03-29T04:46:16.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2151731289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:16.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:15 smithi029 ceph-mon[106624]: pgmap v260: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 7910/78192 objects degraded (10.116%); 0 B/s, 39 objects/s recovering 2024-03-29T04:46:16.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2151731289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:16 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 7910/78192 objects degraded (10.116%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 7910/78192 objects degraded (10.116%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:46:17.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: failed to encode map e98 with expected crc 2024-03-29T04:46:17.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:16 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 7910/78192 objects degraded (10.116%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: pgmap v262: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 7910/70476 objects degraded (11.224%); 0 B/s, 47 objects/s recovering 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:17 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1906833472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: pgmap v262: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 7910/70476 objects degraded (11.224%); 0 B/s, 47 objects/s recovering 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1906833472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: pgmap v262: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 7910/70476 objects degraded (11.224%); 0 B/s, 47 objects/s recovering 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:46:18.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: failed to encode map e99 with expected crc 2024-03-29T04:46:18.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:17 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1906833472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:20.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-mon[102350]: pgmap v264: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 7910/68658 objects degraded (11.521%); 0 B/s, 30 objects/s recovering 2024-03-29T04:46:20.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:20.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:20.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:46:19.747+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:20.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-mon[106624]: pgmap v264: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 7910/68658 objects degraded (11.521%); 0 B/s, 30 objects/s recovering 2024-03-29T04:46:20.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:20.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:20.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:19 smithi099 ceph-mon[94036]: pgmap v264: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 7910/68658 objects degraded (11.521%); 0 B/s, 30 objects/s recovering 2024-03-29T04:46:20.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:20.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:21.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:20 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:21.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:20 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:21.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:20 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3641105910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:21.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:20 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:21.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:20 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:21.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:20 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3641105910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:21.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:20 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:21.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:20 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:21.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:20 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3641105910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:22.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:21 smithi029 ceph-mon[102350]: pgmap v265: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6623/63342 objects degraded (10.456%); 2.5 MiB/s, 61 objects/s recovering 2024-03-29T04:46:22.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:21 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6623/63342 objects degraded (10.456%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:22.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:21 smithi029 ceph-mon[106624]: pgmap v265: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6623/63342 objects degraded (10.456%); 2.5 MiB/s, 61 objects/s recovering 2024-03-29T04:46:22.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:21 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6623/63342 objects degraded (10.456%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:22.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:21 smithi099 ceph-mon[94036]: pgmap v265: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 6623/63342 objects degraded (10.456%); 2.5 MiB/s, 61 objects/s recovering 2024-03-29T04:46:22.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:21 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6623/63342 objects degraded (10.456%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:24.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:23 smithi029 ceph-mon[102350]: pgmap v266: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 6623/58359 objects degraded (11.349%); 2.5 MiB/s, 61 objects/s recovering 2024-03-29T04:46:24.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:23 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/580076587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:24.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:23 smithi029 ceph-mon[106624]: pgmap v266: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 6623/58359 objects degraded (11.349%); 2.5 MiB/s, 61 objects/s recovering 2024-03-29T04:46:24.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:23 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/580076587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:24.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:23 smithi099 ceph-mon[94036]: pgmap v266: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 6623/58359 objects degraded (11.349%); 2.5 MiB/s, 61 objects/s recovering 2024-03-29T04:46:24.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:23 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/580076587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:25.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:46:24] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2024-03-29T04:46:26.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:25 smithi029 ceph-mon[102350]: pgmap v267: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6623/56937 objects degraded (11.632%); 2.3 MiB/s, 57 objects/s recovering 2024-03-29T04:46:26.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3901706606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:26.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:25 smithi029 ceph-mon[106624]: pgmap v267: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6623/56937 objects degraded (11.632%); 2.3 MiB/s, 57 objects/s recovering 2024-03-29T04:46:26.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:25 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3901706606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:26.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:25 smithi099 ceph-mon[94036]: pgmap v267: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 6623/56937 objects degraded (11.632%); 2.3 MiB/s, 57 objects/s recovering 2024-03-29T04:46:26.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:25 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3901706606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:27.192 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:26 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6623/56937 objects degraded (11.632%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:27.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:26 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6623/56937 objects degraded (11.632%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:27.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:26 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6623/56937 objects degraded (11.632%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:28.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:27 smithi029 ceph-mon[102350]: pgmap v268: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 6391/49356 objects degraded (12.949%); 3.4 MiB/s, 72 objects/s recovering 2024-03-29T04:46:28.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:27 smithi029 ceph-mon[106624]: pgmap v268: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 6391/49356 objects degraded (12.949%); 3.4 MiB/s, 72 objects/s recovering 2024-03-29T04:46:28.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:27 smithi099 ceph-mon[94036]: pgmap v268: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 6391/49356 objects degraded (12.949%); 3.4 MiB/s, 72 objects/s recovering 2024-03-29T04:46:29.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:28 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/189633834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:29.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:28 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/189633834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:29.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:28 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/189633834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:30.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:29 smithi029 ceph-mon[102350]: pgmap v269: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 982 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 6391/47190 objects degraded (13.543%); 3.0 MiB/s, 62 objects/s recovering 2024-03-29T04:46:30.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:29 smithi029 ceph-mon[106624]: pgmap v269: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 982 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 6391/47190 objects degraded (13.543%); 3.0 MiB/s, 62 objects/s recovering 2024-03-29T04:46:30.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:29 smithi099 ceph-mon[94036]: pgmap v269: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 982 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 6391/47190 objects degraded (13.543%); 3.0 MiB/s, 62 objects/s recovering 2024-03-29T04:46:31.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:30 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/89219270' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:31.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:30 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/89219270' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:31.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:30 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/89219270' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:32.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:31 smithi099 ceph-mon[94036]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 864 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 6363/41538 objects degraded (15.319%); 3.0 MiB/s, 79 objects/s recovering 2024-03-29T04:46:32.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:31 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6363/41538 objects degraded (15.319%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:32.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:31 smithi029 ceph-mon[102350]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 864 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 6363/41538 objects degraded (15.319%); 3.0 MiB/s, 79 objects/s recovering 2024-03-29T04:46:32.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:31 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6363/41538 objects degraded (15.319%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:32.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:31 smithi029 ceph-mon[106624]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 864 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 6363/41538 objects degraded (15.319%); 3.0 MiB/s, 79 objects/s recovering 2024-03-29T04:46:32.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:31 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6363/41538 objects degraded (15.319%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:34.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:33 smithi099 ceph-mon[94036]: pgmap v271: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 748 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 6363/35955 objects degraded (17.697%); 1.4 MiB/s, 38 objects/s recovering 2024-03-29T04:46:34.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3411755730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:34.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:33 smithi029 ceph-mon[102350]: pgmap v271: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 748 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 6363/35955 objects degraded (17.697%); 1.4 MiB/s, 38 objects/s recovering 2024-03-29T04:46:34.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3411755730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:34.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:33 smithi029 ceph-mon[106624]: pgmap v271: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 748 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 6363/35955 objects degraded (17.697%); 1.4 MiB/s, 38 objects/s recovering 2024-03-29T04:46:34.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3411755730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:35.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:35.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:35.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:46:34] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2024-03-29T04:46:35.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:35.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:35.273 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:35.273 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:35 smithi099 ceph-mon[94036]: pgmap v272: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 720 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 6363/34617 objects degraded (18.381%); 1.4 MiB/s, 38 objects/s recovering 2024-03-29T04:46:36.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:35 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2853423672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:36.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:35 smithi029 ceph-mon[102350]: pgmap v272: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 720 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 6363/34617 objects degraded (18.381%); 1.4 MiB/s, 38 objects/s recovering 2024-03-29T04:46:36.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:35 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2853423672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:36.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:35 smithi029 ceph-mon[106624]: pgmap v272: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 720 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 6363/34617 objects degraded (18.381%); 1.4 MiB/s, 38 objects/s recovering 2024-03-29T04:46:36.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:35 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2853423672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:37.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:36 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6363/34617 objects degraded (18.381%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:37.442 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:36 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6363/34617 objects degraded (18.381%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:37.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:36 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6363/34617 objects degraded (18.381%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:38.218 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:37 smithi099 ceph-mon[94036]: pgmap v273: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 556 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 6363/26757 objects degraded (23.781%); 1.4 MiB/s, 58 objects/s recovering 2024-03-29T04:46:38.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:37 smithi029 ceph-mon[102350]: pgmap v273: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 556 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 6363/26757 objects degraded (23.781%); 1.4 MiB/s, 58 objects/s recovering 2024-03-29T04:46:38.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:37 smithi029 ceph-mon[106624]: pgmap v273: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 556 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 6363/26757 objects degraded (23.781%); 1.4 MiB/s, 58 objects/s recovering 2024-03-29T04:46:39.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:39 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/372389829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:39.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:39 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/372389829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:39.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:39 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/372389829' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:40.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:40 smithi029 ceph-mon[102350]: pgmap v274: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 514 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 6363/24723 objects degraded (25.737%); 149 KiB/s, 38 objects/s recovering 2024-03-29T04:46:40.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:40.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:40 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:40.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:40 smithi029 ceph-mon[106624]: pgmap v274: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 514 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 6363/24723 objects degraded (25.737%); 149 KiB/s, 38 objects/s recovering 2024-03-29T04:46:40.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:40 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:40.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:40 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:40.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:40 smithi099 ceph-mon[94036]: pgmap v274: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 514 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 6363/24723 objects degraded (25.737%); 149 KiB/s, 38 objects/s recovering 2024-03-29T04:46:40.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:40 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:40.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:40 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:42 smithi099 ceph-mon[94036]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 414 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6363/19917 objects degraded (31.948%); 149 KiB/s, 60 objects/s recovering 2024-03-29T04:46:42.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:42 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3992447954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:42.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:42 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:42 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:42 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6363/19917 objects degraded (31.948%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[102350]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 414 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6363/19917 objects degraded (31.948%); 149 KiB/s, 60 objects/s recovering 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3992447954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6363/19917 objects degraded (31.948%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[106624]: pgmap v275: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 414 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 6363/19917 objects degraded (31.948%); 149 KiB/s, 60 objects/s recovering 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3992447954' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:42.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6363/19917 objects degraded (31.948%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:42.943 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:42 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:46:42.469+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:43.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:43.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:43.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:43.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:43.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:43.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:43.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:43.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:44.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:44 smithi099 ceph-mon[94036]: pgmap v276: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 312 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 6363/15015 objects degraded (42.378%); 0 B/s, 42 objects/s recovering 2024-03-29T04:46:44.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:44 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:44.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:44 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:44.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:44 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/602643319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:44.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[102350]: pgmap v276: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 312 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 6363/15015 objects degraded (42.378%); 0 B/s, 42 objects/s recovering 2024-03-29T04:46:44.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:44.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:44.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/602643319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:44.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[106624]: pgmap v276: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 312 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 6363/15015 objects degraded (42.378%); 0 B/s, 42 objects/s recovering 2024-03-29T04:46:44.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:44.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:46:44.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/602643319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:45.193 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:46:44] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2024-03-29T04:46:46.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:46 smithi099 ceph-mon[94036]: pgmap v277: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 284 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 6363/13692 objects degraded (46.472%); 0 B/s, 42 objects/s recovering 2024-03-29T04:46:46.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:46 smithi029 ceph-mon[102350]: pgmap v277: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 284 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 6363/13692 objects degraded (46.472%); 0 B/s, 42 objects/s recovering 2024-03-29T04:46:46.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:46 smithi029 ceph-mon[106624]: pgmap v277: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 284 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 6363/13692 objects degraded (46.472%); 0 B/s, 42 objects/s recovering 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: osdmap e100: 8 total, 8 up, 8 in 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2413201001' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:47.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:47 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6363/13692 objects degraded (46.472%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: osdmap e100: 8 total, 8 up, 8 in 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2413201001' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6363/13692 objects degraded (46.472%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: osdmap e100: 8 total, 8 up, 8 in 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: failed to encode map e100 with expected crc 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2413201001' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:47 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6363/13692 objects degraded (46.472%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:48.043 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 117591 objects 2024-03-29T04:46:48.044 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :159.621 2024-03-29T04:46:48.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: pgmap v279: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 128 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 6363/6204 objects degraded (102.563%); 0 B/s, 51 objects/s recovering 2024-03-29T04:46:48.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: osdmap e101: 8 total, 8 up, 8 in 2024-03-29T04:46:48.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:48 smithi099 ceph-mon[94036]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: pgmap v279: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 128 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 6363/6204 objects degraded (102.563%); 0 B/s, 51 objects/s recovering 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: osdmap e101: 8 total, 8 up, 8 in 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[106624]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: pgmap v279: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 128 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 6363/6204 objects degraded (102.563%); 0 B/s, 51 objects/s recovering 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: osdmap e101: 8 total, 8 up, 8 in 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:48 smithi029 ceph-mon[102350]: failed to encode map e101 with expected crc 2024-03-29T04:46:48.512 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-03-29T04:46:48.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-03-29T04:46:48.818 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:49.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:49 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/823349493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:49.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:49 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/823349493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:49.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:49.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:49 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/823349493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:49.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:50.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:50 smithi099 ceph-mon[94036]: pgmap v281: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 89 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 6363/4332 objects degraded (146.884%); 0 B/s, 31 objects/s recovering 2024-03-29T04:46:50.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:50 smithi029 ceph-mon[102350]: pgmap v281: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 89 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 6363/4332 objects degraded (146.884%); 0 B/s, 31 objects/s recovering 2024-03-29T04:46:50.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:50 smithi029 ceph-mon[106624]: pgmap v281: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 89 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 6363/4332 objects degraded (146.884%); 0 B/s, 31 objects/s recovering 2024-03-29T04:46:51.563 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_2' removed 2024-03-29T04:46:52.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: pgmap v282: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 32 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 6363/1560 objects degraded (407.885%); 0 B/s, 33 objects/s recovering 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2199275117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4015159537' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6363/1560 objects degraded (407.885%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: failed to encode map e102 with expected crc 2024-03-29T04:46:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:52 smithi099 ceph-mon[94036]: osdmap e102: 8 total, 8 up, 8 in 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: pgmap v282: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 32 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 6363/1560 objects degraded (407.885%); 0 B/s, 33 objects/s recovering 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2199275117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4015159537' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6363/1560 objects degraded (407.885%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: failed to encode map e102 with expected crc 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[102350]: osdmap e102: 8 total, 8 up, 8 in 2024-03-29T04:46:52.392 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: pgmap v282: 89 pgs: 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 32 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 6363/1560 objects degraded (407.885%); 0 B/s, 33 objects/s recovering 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2199275117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4015159537' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6363/1560 objects degraded (407.885%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: failed to encode map e102 with expected crc 2024-03-29T04:46:52.393 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:52 smithi029 ceph-mon[106624]: osdmap e102: 8 total, 8 up, 8 in 2024-03-29T04:46:52.415 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-03-29T04:46:52.415 INFO:tasks.radosbench:Beginning radosbench... 2024-03-29T04:46:52.416 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-03-29T04:46:52.416 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool create unique_pool_3 16 2024-03-29T04:46:52.677 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:53.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:53 smithi099 ceph-mon[94036]: failed to encode map e102 with expected crc 2024-03-29T04:46:53.408 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:53 smithi029 ceph-mon[102350]: failed to encode map e102 with expected crc 2024-03-29T04:46:53.408 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:53 smithi029 ceph-mon[106624]: failed to encode map e102 with expected crc 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:54 smithi099 ceph-mon[94036]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:54 smithi099 ceph-mon[94036]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6363/1560 objects degraded (407.885%), 4 pgs degraded, 4 pgs undersized) 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:54 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3145764469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:54.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-mon[102350]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:54.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-mon[102350]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6363/1560 objects degraded (407.885%), 4 pgs degraded, 4 pgs undersized) 2024-03-29T04:46:54.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3145764469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:54.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-mon[106624]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:54.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-mon[106624]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6363/1560 objects degraded (407.885%), 4 pgs degraded, 4 pgs undersized) 2024-03-29T04:46:54.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3145764469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:55.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:55 smithi029 ceph-mon[102350]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:55.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:55 smithi029 ceph-mon[106624]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:55.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:46:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:46:54] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.43.0" 2024-03-29T04:46:55.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:55 smithi099 ceph-mon[94036]: pgmap v285: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:56.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:56 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3030664455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:56.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:56 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3030664455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:56.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:56 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3030664455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:57.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:57 smithi099 ceph-mon[94036]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:57.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:57 smithi029 ceph-mon[102350]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:57.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:57 smithi029 ceph-mon[106624]: pgmap v286: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:58.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:58 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:58.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:58 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:58.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:58 smithi099 ceph-mon[94036]: Upgrade: osd.1 is safe to restart 2024-03-29T04:46:58.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:58.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:58 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:58.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:58 smithi029 ceph-mon[102350]: Upgrade: osd.1 is safe to restart 2024-03-29T04:46:58.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:58 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:58.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:58 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:46:58.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:58 smithi029 ceph-mon[106624]: Upgrade: osd.1 is safe to restart 2024-03-29T04:46:59.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:59 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1651330331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:59.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:46:59 smithi099 ceph-mon[94036]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:59.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:59 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1651330331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:59.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:46:59 smithi029 ceph-mon[102350]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:59.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:59 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1651330331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:46:59.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:46:59 smithi029 ceph-mon[106624]: pgmap v287: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:00.301 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:00 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:00.301 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:00 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:00.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:00 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:01.087 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_3' created 2024-03-29T04:47:01.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3715028917' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-03-29T04:47:01.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:01 smithi029 ceph-mon[106624]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:01.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2940812497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:01.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3715028917' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-03-29T04:47:01.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:01 smithi029 ceph-mon[102350]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:01.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2940812497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:01.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3715028917' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-03-29T04:47:01.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:01 smithi099 ceph-mon[94036]: pgmap v288: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:01.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2940812497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:02.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3715028917' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-03-29T04:47:02.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.193 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: Upgrade: Updating osd.1 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[106624]: Deploying daemon osd.1 on smithi029 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3715028917' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-03-29T04:47:02.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: Upgrade: Updating osd.1 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:02 smithi029 ceph-mon[102350]: Deploying daemon osd.1 on smithi029 2024-03-29T04:47:02.220 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-03-29T04:47:02.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3715028917' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: failed to encode map e103 with expected crc 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: Upgrade: Updating osd.1 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:02.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:02.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:02 smithi099 ceph-mon[94036]: Deploying daemon osd.1 on smithi029 2024-03-29T04:47:03.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: pgmap v291: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/489765234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: pgmap v291: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/489765234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:03.219 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:03.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: pgmap v291: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: failed to encode map e104 with expected crc 2024-03-29T04:47:03.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/489765234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:04.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:47:04.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: osdmap e105: 8 total, 8 up, 8 in 2024-03-29T04:47:04.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:04.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:04.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:47:04.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: osdmap e105: 8 total, 8 up, 8 in 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: osdmap e105: 8 total, 8 up, 8 in 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: failed to encode map e105 with expected crc 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:04.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:05.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:47:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:47:04] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.43.0" 2024-03-29T04:47:05.176 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[102350]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[102350]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[102350]: pgmap v293: 89 pgs: 3 creating+peering, 10 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[102350]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[106624]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[106624]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[106624]: pgmap v293: 89 pgs: 3 creating+peering, 10 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:05.177 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:05 smithi029 ceph-mon[106624]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:05 smithi099 ceph-mon[94036]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:05 smithi099 ceph-mon[94036]: failed to encode map e105 with expected crc 2024-03-29T04:47:05.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:05 smithi099 ceph-mon[94036]: pgmap v293: 89 pgs: 3 creating+peering, 10 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:05.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:05 smithi099 ceph-mon[94036]: failed to encode map e105 with expected crc 2024-03-29T04:47:06.108 INFO:teuthology.orchestra.run.smithi029.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-03-29T04:47:06.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1785792440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:06.374 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/672434450' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:47:06.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1785792440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:06.374 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/672434450' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:47:06.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1785792440' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:06.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/672434450' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/672434450' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[102350]: failed to encode map e106 with expected crc 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[102350]: failed to encode map e106 with expected crc 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[102350]: osdmap e106: 8 total, 8 up, 8 in 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[102350]: pgmap v295: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/672434450' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:47:07.295 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[106624]: failed to encode map e106 with expected crc 2024-03-29T04:47:07.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[106624]: failed to encode map e106 with expected crc 2024-03-29T04:47:07.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[106624]: osdmap e106: 8 total, 8 up, 8 in 2024-03-29T04:47:07.296 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:07 smithi029 ceph-mon[106624]: pgmap v295: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:07.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:07 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/672434450' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:47:07.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:07 smithi099 ceph-mon[94036]: failed to encode map e106 with expected crc 2024-03-29T04:47:07.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:07 smithi099 ceph-mon[94036]: failed to encode map e106 with expected crc 2024-03-29T04:47:07.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:07 smithi099 ceph-mon[94036]: osdmap e106: 8 total, 8 up, 8 in 2024-03-29T04:47:07.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:07 smithi099 ceph-mon[94036]: pgmap v295: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:07.693 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:07 smithi029 systemd[1]: Stopping Ceph osd.1 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:47:08.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:08 smithi099 ceph-mon[94036]: failed to encode map e106 with expected crc 2024-03-29T04:47:08.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:08 smithi099 ceph-mon[94036]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:47:08.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:08 smithi099 ceph-mon[94036]: failed to encode map e106 with expected crc 2024-03-29T04:47:08.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:08 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/199232558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:08.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:08 smithi099 ceph-mon[94036]: osd.1 marked itself down and dead 2024-03-29T04:47:08.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[102350]: failed to encode map e106 with expected crc 2024-03-29T04:47:08.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[102350]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[102350]: failed to encode map e106 with expected crc 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/199232558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[102350]: osd.1 marked itself down and dead 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[106624]: failed to encode map e106 with expected crc 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[106624]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[106624]: failed to encode map e106 with expected crc 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/199232558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:08.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-mon[106624]: osd.1 marked itself down and dead 2024-03-29T04:47:08.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[52463]: 2024-03-29T04:47:08.031+0000 7fab33a6c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:47:08.445 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[52463]: 2024-03-29T04:47:08.031+0000 7fab33a6c700 -1 osd.1 106 *** Got signal Terminated *** 2024-03-29T04:47:08.445 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:08 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[52463]: 2024-03-29T04:47:08.031+0000 7fab33a6c700 -1 osd.1 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:47:08.759 DEBUG:teuthology.orchestra.run.smithi029:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-03-29T04:47:08.762 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-03-29T04:47:08.762 INFO:teuthology.orchestra.run:waiting for 3000 2024-03-29T04:47:08.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:hints = 1 2024-03-29T04:47:08.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-03-29T04:47:08.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object prefix: benchmark_data_smithi029_124194 2024-03-29T04:47:08.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:47:08.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 0 0 0 0 0 0 - 0 2024-03-29T04:47:08.866 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:08 smithi029 podman[124153]: 2024-03-29 04:47:08.5412727 +0000 UTC m=+0.648186800 container died 6532f8ba40e947221b49b3684e8cee06a0fbbd1cb5eff0c9f3210349399b0bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.license=GPLv2) 2024-03-29T04:47:09.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:47:09.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: osdmap e107: 8 total, 7 up, 8 in 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[102350]: pgmap v297: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:09.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:08 smithi029 podman[124153]: 2024-03-29 04:47:08.957764625 +0000 UTC m=+1.064678712 container cleanup 6532f8ba40e947221b49b3684e8cee06a0fbbd1cb5eff0c9f3210349399b0bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-29T04:47:09.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:08 smithi029 bash[124153]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: osdmap e107: 8 total, 7 up, 8 in 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:09 smithi029 ceph-mon[106624]: pgmap v297: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:09.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:47:09.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: osdmap e107: 8 total, 7 up, 8 in 2024-03-29T04:47:09.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: failed to encode map e107 with expected crc 2024-03-29T04:47:09.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:09 smithi099 ceph-mon[94036]: pgmap v297: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:09.693 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:09 smithi029 podman[124176]: 2024-03-29 04:47:09.244309475 +0000 UTC m=+0.697662738 container remove 6532f8ba40e947221b49b3684e8cee06a0fbbd1cb5eff0c9f3210349399b0bbb (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-29T04:47:09.807 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 1 16 2950 2934 183.369 183.375 0.0024157 0.00543738 2024-03-29T04:47:10.120 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:09 smithi029 podman[124302]: 2024-03-29 04:47:09.601268322 +0000 UTC m=+0.036132186 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:10.120 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:09 smithi029 podman[124302]: 2024-03-29 04:47:09.732088647 +0000 UTC m=+0.166952491 container create 69dffbb9979b87d605c5f0d152f4e235e39b4c614d84ba96d24f410af56e67f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:47:10.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: osdmap e108: 8 total, 7 up, 8 in 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:10 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2873306240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:10.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: osdmap e108: 8 total, 7 up, 8 in 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2873306240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: osdmap e108: 8 total, 7 up, 8 in 2024-03-29T04:47:10.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: failed to encode map e108 with expected crc 2024-03-29T04:47:10.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:10 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2873306240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:10.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 2 16 6498 6482 202.55 221.75 0.00226009 0.00492859 2024-03-29T04:47:10.944 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:10 smithi029 podman[124302]: 2024-03-29 04:47:10.588403801 +0000 UTC m=+1.023267652 container init 69dffbb9979b87d605c5f0d152f4e235e39b4c614d84ba96d24f410af56e67f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:47:10.944 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:10 smithi029 podman[124302]: 2024-03-29 04:47:10.599392851 +0000 UTC m=+1.034256702 container start 69dffbb9979b87d605c5f0d152f4e235e39b4c614d84ba96d24f410af56e67f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362) 2024-03-29T04:47:10.944 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:10 smithi029 podman[124302]: 2024-03-29 04:47:10.698759055 +0000 UTC m=+1.133622901 container attach 69dffbb9979b87d605c5f0d152f4e235e39b4c614d84ba96d24f410af56e67f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-29T04:47:11.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:11 smithi029 ceph-mon[102350]: pgmap v299: 89 pgs: 3 active+undersized+degraded, 5 peering, 6 stale+active+clean, 16 active+undersized, 59 active+clean; 100 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 266 op/s; 204/4863 objects degraded (4.195%) 2024-03-29T04:47:11.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:11 smithi029 ceph-mon[106624]: pgmap v299: 89 pgs: 3 active+undersized+degraded, 5 peering, 6 stale+active+clean, 16 active+undersized, 59 active+clean; 100 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 266 op/s; 204/4863 objects degraded (4.195%) 2024-03-29T04:47:11.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:11 smithi029 podman[124302]: 2024-03-29 04:47:11.150218854 +0000 UTC m=+1.585082708 container died 69dffbb9979b87d605c5f0d152f4e235e39b4c614d84ba96d24f410af56e67f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-29T04:47:11.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:11 smithi099 ceph-mon[94036]: pgmap v299: 89 pgs: 3 active+undersized+degraded, 5 peering, 6 stale+active+clean, 16 active+undersized, 59 active+clean; 100 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s wr, 266 op/s; 204/4863 objects degraded (4.195%) 2024-03-29T04:47:11.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 3 16 10532 10516 219.067 252.125 0.00530419 0.0045581 2024-03-29T04:47:12.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:12 smithi029 ceph-mon[102350]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:47:12.354 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:12 smithi029 ceph-mon[102350]: Health check failed: Degraded data redundancy: 204/4863 objects degraded (4.195%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:12.354 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:12 smithi029 ceph-mon[106624]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:47:12.354 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:12 smithi029 ceph-mon[106624]: Health check failed: Degraded data redundancy: 204/4863 objects degraded (4.195%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:12.355 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:11 smithi029 podman[124370]: 2024-03-29 04:47:11.986307514 +0000 UTC m=+0.826284611 container remove 69dffbb9979b87d605c5f0d152f4e235e39b4c614d84ba96d24f410af56e67f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS) 2024-03-29T04:47:12.355 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-29T04:47:12.355 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 systemd[1]: Stopped Ceph osd.1 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:47:12.355 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.1.service: Consumed 15min 35.564s CPU time. 2024-03-29T04:47:12.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:12 smithi099 ceph-mon[94036]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:47:12.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:12 smithi099 ceph-mon[94036]: Health check failed: Degraded data redundancy: 204/4863 objects degraded (4.195%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:12.670 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 systemd[1]: Starting Ceph osd.1 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:47:12.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 4 16 13952 13936 217.733 213.75 0.00345662 0.00458643 2024-03-29T04:47:12.943 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 podman[124487]: 2024-03-29 04:47:12.57438586 +0000 UTC m=+0.037286281 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:12.944 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 podman[124487]: 2024-03-29 04:47:12.713695127 +0000 UTC m=+0.176595541 container create 5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-29T04:47:13.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-mon[102350]: pgmap v300: 89 pgs: 8 active+undersized+degraded, 5 peering, 26 active+undersized, 50 active+clean; 307 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 3505/14766 objects degraded (23.737%) 2024-03-29T04:47:13.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/203680318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:13.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-mon[106624]: pgmap v300: 89 pgs: 8 active+undersized+degraded, 5 peering, 26 active+undersized, 50 active+clean; 307 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 3505/14766 objects degraded (23.737%) 2024-03-29T04:47:13.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/203680318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:13.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 podman[124487]: 2024-03-29 04:47:12.979083273 +0000 UTC m=+0.441983689 container init 5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362) 2024-03-29T04:47:13.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:12 smithi029 podman[124487]: 2024-03-29 04:47:12.987849465 +0000 UTC m=+0.450749877 container start 5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5) 2024-03-29T04:47:13.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 podman[124487]: 2024-03-29 04:47:13.079525176 +0000 UTC m=+0.542425588 container attach 5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-03-29T04:47:13.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:13 smithi099 ceph-mon[94036]: pgmap v300: 89 pgs: 8 active+undersized+degraded, 5 peering, 26 active+undersized, 50 active+clean; 307 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 790 op/s; 3505/14766 objects degraded (23.737%) 2024-03-29T04:47:13.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:13 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/203680318' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:13.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 5 16 17347 17331 216.621 212.188 0.00677818 0.00461003 2024-03-29T04:47:14.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:47:14.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-29T04:47:14.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:47:14.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-29T04:47:14.194 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 bash[124487]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate[124504]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 conmon[124504]: conmon 5f988aad91ec20f75e29 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed.scope/memory.events 2024-03-29T04:47:14.195 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:13 smithi029 podman[124487]: 2024-03-29 04:47:13.826631807 +0000 UTC m=+1.289532235 container died 5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-29T04:47:14.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 6 16 20358 20342 211.879 188.188 0.00643629 0.00471425 2024-03-29T04:47:15.051 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:47:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:47:14] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.43.0" 2024-03-29T04:47:15.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:15 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:15.367 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:15 smithi029 ceph-mon[102350]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 596 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 4641/28635 objects degraded (16.207%) 2024-03-29T04:47:15.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:15 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:15.368 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:15 smithi029 ceph-mon[106624]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 596 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 4641/28635 objects degraded (16.207%) 2024-03-29T04:47:15.368 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:15 smithi029 podman[124656]: 2024-03-29 04:47:15.050341875 +0000 UTC m=+1.213062653 container remove 5f988aad91ec20f75e2914e7083f01c8bfa97b8108b993f0f18d02b2dec5b3ed (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-29T04:47:15.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:15 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:15.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:15 smithi099 ceph-mon[94036]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 596 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 4641/28635 objects degraded (16.207%) 2024-03-29T04:47:15.693 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:15 smithi029 podman[124736]: 2024-03-29 04:47:15.272965225 +0000 UTC m=+0.044193173 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:15.693 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:15 smithi029 podman[124736]: 2024-03-29 04:47:15.412807395 +0000 UTC m=+0.184035340 container create cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:47:15.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 7 16 23029 23013 205.456 166.938 0.00425478 0.00483809 2024-03-29T04:47:16.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:16 smithi099 ceph-mon[94036]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-29T04:47:16.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:16 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2984730116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:16.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:16 smithi029 ceph-mon[102350]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-29T04:47:16.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:16 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2984730116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:16.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:16 smithi029 ceph-mon[106624]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-29T04:47:16.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:16 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2984730116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:16.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:16 smithi029 podman[124736]: 2024-03-29 04:47:16.111262881 +0000 UTC m=+0.882490840 container init cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T04:47:16.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:16 smithi029 podman[124736]: 2024-03-29 04:47:16.119155455 +0000 UTC m=+0.890383393 container start cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:47:16.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:16 smithi029 ceph-osd[124776]: -- 172.21.15.29:0/3627331152 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562ed374b680 con 0x562ed3715400 2024-03-29T04:47:16.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:16 smithi029 bash[124736]: cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 2024-03-29T04:47:16.444 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:16 smithi029 systemd[1]: Started Ceph osd.1 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:47:16.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 8 16 25712 25696 200.733 167.688 0.00799567 0.00497622 2024-03-29T04:47:16.943 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:16 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[124752]: 2024-03-29T04:47:16.674+0000 7f67c9be77c0 -1 Falling back to public interface 2024-03-29T04:47:17.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: pgmap v302: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 6769/54354 objects degraded (12.454%) 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 4641/28635 objects degraded (16.207%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:17 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: pgmap v302: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 6769/54354 objects degraded (12.454%) 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 4641/28635 objects degraded (16.207%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: pgmap v302: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 6769/54354 objects degraded (12.454%) 2024-03-29T04:47:17.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 4641/28635 objects degraded (16.207%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:17 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:17.808 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 9 16 28202 28186 195.72 155.625 0.00391855 0.0051031 2024-03-29T04:47:18.579 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:18 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3985229887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:18.579 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:18 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3985229887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:18.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:18 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3985229887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:18.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 10 16 30724 30708 191.867 157.625 0.00867847 0.00520705 2024-03-29T04:47:19.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 11 16 33165 33149 188.293 152.562 0.0077989 0.00530583 2024-03-29T04:47:20.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:19 smithi099 ceph-mon[94036]: pgmap v303: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s; 10523/65616 objects degraded (16.037%) 2024-03-29T04:47:20.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:20.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:20.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:19 smithi029 ceph-mon[102350]: pgmap v303: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s; 10523/65616 objects degraded (16.037%) 2024-03-29T04:47:20.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:20.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:20.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:19 smithi029 ceph-mon[106624]: pgmap v303: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s; 10523/65616 objects degraded (16.037%) 2024-03-29T04:47:20.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:20.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:20.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 12 15 35089 35074 182.628 120.312 0.00731635 0.00547098 2024-03-29T04:47:21.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:20 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1830056546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:21.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:20 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1830056546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:21.443 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:20 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1830056546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:21.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 13 16 36472 36456 175.225 86.375 0.00255949 0.00563467 2024-03-29T04:47:22.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:21 smithi099 ceph-mon[94036]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 171 MiB/s wr, 2.73k op/s; 12158/91794 objects degraded (13.245%) 2024-03-29T04:47:22.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:21 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 12158/91794 objects degraded (13.245%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:22.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:21 smithi029 ceph-mon[102350]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 171 MiB/s wr, 2.73k op/s; 12158/91794 objects degraded (13.245%) 2024-03-29T04:47:22.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:21 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 12158/91794 objects degraded (13.245%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:22.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:21 smithi029 ceph-mon[106624]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 171 MiB/s wr, 2.73k op/s; 12158/91794 objects degraded (13.245%) 2024-03-29T04:47:22.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:21 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 12158/91794 objects degraded (13.245%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:22.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 14 16 37989 37973 169.481 94.8125 0.0023341 0.00588562 2024-03-29T04:47:22.991 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[124752]: 2024-03-29T04:47:22.690+0000 7f67c9be77c0 -1 osd.1 106 log_to_monitors true 2024-03-29T04:47:23.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:22 smithi029 ceph-mon[102350]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:23.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:22 smithi029 ceph-mon[106624]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:23.373 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:22 smithi099 ceph-mon[94036]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:23.693 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:47:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[124752]: 2024-03-29T04:47:23.347+0000 7f67b7037700 -1 osd.1 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:47:23.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 15 16 39924 39908 166.245 120.938 0.00456224 0.00600998 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:23 smithi029 ceph-mon[102350]: pgmap v305: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 168 MiB/s wr, 2.69k op/s; 15505/101835 objects degraded (15.226%) 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: osdmap e109: 8 total, 7 up, 8 in 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2287848778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:24.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:23 smithi029 ceph-mon[106624]: pgmap v305: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 168 MiB/s wr, 2.69k op/s; 15505/101835 objects degraded (15.226%) 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: osdmap e109: 8 total, 7 up, 8 in 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2287848778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:24.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: pgmap v305: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.1 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 168 MiB/s wr, 2.69k op/s; 15505/101835 objects degraded (15.226%) 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: osdmap e109: 8 total, 7 up, 8 in 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: failed to encode map e109 with expected crc 2024-03-29T04:47:24.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2287848778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:24.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:23 smithi099 ceph-mon[94036]: from='osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000]' entity='osd.1' 2024-03-29T04:47:24.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 16 16 40308 40292 157.355 24 0.00393109 0.00605423 2024-03-29T04:47:25.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[102350]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:47:25.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-mon[106624]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:47:25.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:47:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:47:24] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000] boot 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: osdmap e110: 8 total, 8 up, 8 in 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:24 smithi099 ceph-mon[94036]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000] boot 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: osdmap e110: 8 total, 8 up, 8 in 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: osd.1 [v2:172.21.15.29:6810/3180340000,v1:172.21.15.29:6811/3180340000] boot 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: osdmap e110: 8 total, 8 up, 8 in 2024-03-29T04:47:25.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:25.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e110 with expected crc 2024-03-29T04:47:25.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 17 16 40308 40292 148.1 0 - 0.00605423 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: pgmap v308: 89 pgs: 3 peering, 9 active+undersized+degraded, 27 active+undersized, 50 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s; 15504/106734 objects degraded (14.526%) 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: osdmap e111: 8 total, 8 up, 8 in 2024-03-29T04:47:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:25 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1765430841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: pgmap v308: 89 pgs: 3 peering, 9 active+undersized+degraded, 27 active+undersized, 50 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s; 15504/106734 objects degraded (14.526%) 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: osdmap e111: 8 total, 8 up, 8 in 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1765430841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: pgmap v308: 89 pgs: 3 peering, 9 active+undersized+degraded, 27 active+undersized, 50 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s; 15504/106734 objects degraded (14.526%) 2024-03-29T04:47:26.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: osdmap e111: 8 total, 8 up, 8 in 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: failed to encode map e111 with expected crc 2024-03-29T04:47:26.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:25 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1765430841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:26.811 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 18 16 40869 40853 141.821 17.5312 0.00777344 0.00704421 2024-03-29T04:47:27.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: osdmap e112: 8 total, 8 up, 8 in 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: pgmap v311: 89 pgs: 1 remapped+peering, 17 peering, 6 active+undersized+degraded, 14 active+undersized, 51 active+clean; 2.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 13391/115245 objects degraded (11.620%) 2024-03-29T04:47:27.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:27 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 15504/106734 objects degraded (14.526%), 9 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: osdmap e112: 8 total, 8 up, 8 in 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: pgmap v311: 89 pgs: 1 remapped+peering, 17 peering, 6 active+undersized+degraded, 14 active+undersized, 51 active+clean; 2.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 13391/115245 objects degraded (11.620%) 2024-03-29T04:47:27.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 15504/106734 objects degraded (14.526%), 9 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: osdmap e112: 8 total, 8 up, 8 in 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: failed to encode map e112 with expected crc 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: pgmap v311: 89 pgs: 1 remapped+peering, 17 peering, 6 active+undersized+degraded, 14 active+undersized, 51 active+clean; 2.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 13391/115245 objects degraded (11.620%) 2024-03-29T04:47:27.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:27 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 15504/106734 objects degraded (14.526%), 9 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:27.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 19 15 42156 42141 138.593 80.5 0.00617528 0.0072065 2024-03-29T04:47:28.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:28 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1770411192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:28.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:28 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1770411192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:28.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:28 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1770411192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:28.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:47:28.811701+0000 min lat: 0.000920881 max lat: 2.40197 avg lat: 0.00735962 2024-03-29T04:47:28.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:47:28.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 20 16 43470 43454 135.767 82.0625 0.00456961 0.00735962 2024-03-29T04:47:29.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:29 smithi099 ceph-mon[94036]: pgmap v312: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 17 peering, 2 active+undersized+degraded, 65 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 14253/118497 objects degraded (12.028%) 2024-03-29T04:47:29.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:29 smithi029 ceph-mon[102350]: pgmap v312: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 17 peering, 2 active+undersized+degraded, 65 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 14253/118497 objects degraded (12.028%) 2024-03-29T04:47:29.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:29 smithi029 ceph-mon[106624]: pgmap v312: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 17 peering, 2 active+undersized+degraded, 65 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 14253/118497 objects degraded (12.028%) 2024-03-29T04:47:29.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 21 16 44811 44795 133.292 83.8125 0.00233614 0.0074938 2024-03-29T04:47:30.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 22 16 45625 45609 129.546 50.875 0.0110539 0.00771013 2024-03-29T04:47:31.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:31 smithi099 ceph-mon[94036]: pgmap v313: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 15218/130527 objects degraded (11.659%); 819 KiB/s, 12 objects/s recovering 2024-03-29T04:47:31.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:31 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/296124914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:31.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:31 smithi029 ceph-mon[102350]: pgmap v313: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 15218/130527 objects degraded (11.659%); 819 KiB/s, 12 objects/s recovering 2024-03-29T04:47:31.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:31 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/296124914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:31.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:31 smithi029 ceph-mon[106624]: pgmap v313: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 15218/130527 objects degraded (11.659%); 819 KiB/s, 12 objects/s recovering 2024-03-29T04:47:31.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:31 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/296124914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:31.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 23 16 46453 46437 126.164 51.75 0.00566005 0.00790825 2024-03-29T04:47:32.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:32 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 15218/130527 objects degraded (11.659%), 6 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:32.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:32 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 15218/130527 objects degraded (11.659%), 6 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:32.693 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:32 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 15218/130527 objects degraded (11.659%), 6 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:32.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 24 16 47417 47401 123.418 60.25 0.044165 0.00809725 2024-03-29T04:47:33.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:33 smithi099 ceph-mon[94036]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 17677/135462 objects degraded (13.049%); 640 KiB/s, 9 objects/s recovering 2024-03-29T04:47:33.624 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/10460849' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:33.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:33 smithi029 ceph-mon[102350]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 17677/135462 objects degraded (13.049%); 640 KiB/s, 9 objects/s recovering 2024-03-29T04:47:33.693 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/10460849' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:33.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:33 smithi029 ceph-mon[106624]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 17677/135462 objects degraded (13.049%); 640 KiB/s, 9 objects/s recovering 2024-03-29T04:47:33.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/10460849' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:33.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 25 16 48403 48387 120.946 61.625 0.0151366 0.00826094 2024-03-29T04:47:34.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 26 16 49311 49295 118.477 56.75 0.0227154 0.00843182 2024-03-29T04:47:35.193 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:35.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:35.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:47:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:47:34] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-03-29T04:47:35.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:35.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:35.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:35.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:35.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 27 16 50202 50186 116.151 55.6875 0.0167916 0.00860369 2024-03-29T04:47:36.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:35 smithi099 ceph-mon[94036]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 17557/138966 objects degraded (12.634%); 1.4 MiB/s, 22 objects/s recovering 2024-03-29T04:47:36.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:35 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2546743807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:36.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:35 smithi029 ceph-mon[102350]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 17557/138966 objects degraded (12.634%); 1.4 MiB/s, 22 objects/s recovering 2024-03-29T04:47:36.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:35 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2546743807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:36.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:35 smithi029 ceph-mon[106624]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 17557/138966 objects degraded (12.634%); 1.4 MiB/s, 22 objects/s recovering 2024-03-29T04:47:36.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:35 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2546743807' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:36.812 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 28 16 51221 51205 114.278 63.6875 0.00875486 0.0087435 2024-03-29T04:47:37.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:36 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 17557/138966 objects degraded (12.634%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:37.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:36 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 17557/138966 objects degraded (12.634%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:37.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:36 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 17557/138966 objects degraded (12.634%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:37.813 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 29 16 52011 51995 112.037 49.375 0.00975273 0.00891845 2024-03-29T04:47:38.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:37 smithi099 ceph-mon[94036]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 17557/145908 objects degraded (12.033%); 1.2 MiB/s, 19 objects/s recovering 2024-03-29T04:47:38.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:37 smithi029 ceph-mon[102350]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 17557/145908 objects degraded (12.033%); 1.2 MiB/s, 19 objects/s recovering 2024-03-29T04:47:38.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:37 smithi029 ceph-mon[106624]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 17557/145908 objects degraded (12.033%); 1.2 MiB/s, 19 objects/s recovering 2024-03-29T04:47:38.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 30 16 52678 52662 109.692 41.6875 0.00809171 0.00910996 2024-03-29T04:47:39.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3757994077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:39.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3757994077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:39.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3757994077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:39.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 31 16 53651 53635 108.115 60.8125 0.0186528 0.00924387 2024-03-29T04:47:40.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:39 smithi099 ceph-mon[94036]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 17557/149412 objects degraded (11.751%); 1.0 MiB/s, 16 objects/s recovering 2024-03-29T04:47:40.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:39 smithi029 ceph-mon[102350]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 17557/149412 objects degraded (11.751%); 1.0 MiB/s, 16 objects/s recovering 2024-03-29T04:47:40.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:39 smithi029 ceph-mon[106624]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s; 17557/149412 objects degraded (11.751%); 1.0 MiB/s, 16 objects/s recovering 2024-03-29T04:47:40.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 32 16 54304 54288 106.012 40.8125 0.0152252 0.00942662 2024-03-29T04:47:41.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:40 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/988498862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:41.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:40 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/988498862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:41.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:40 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/988498862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:41.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 33 16 55318 55302 104.72 63.375 0.00531324 0.00954261 2024-03-29T04:47:42.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:42 smithi099 ceph-mon[94036]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 17419/158331 objects degraded (11.002%); 1.8 MiB/s, 28 objects/s recovering 2024-03-29T04:47:42.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:42 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 17419/158331 objects degraded (11.002%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:42 smithi029 ceph-mon[102350]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 17419/158331 objects degraded (11.002%); 1.8 MiB/s, 28 objects/s recovering 2024-03-29T04:47:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:42 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 17419/158331 objects degraded (11.002%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:42.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:42 smithi029 ceph-mon[106624]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 17419/158331 objects degraded (11.002%); 1.8 MiB/s, 28 objects/s recovering 2024-03-29T04:47:42.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:42 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 17419/158331 objects degraded (11.002%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:42.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 34 16 56234 56218 103.324 57.25 0.00401109 0.00967167 2024-03-29T04:47:43.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 35 16 57156 57140 102.018 57.625 0.00320582 0.0097749 2024-03-29T04:47:44.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:44 smithi099 ceph-mon[94036]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 17419/162174 objects degraded (10.741%); 1.3 MiB/s, 21 objects/s recovering 2024-03-29T04:47:44.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:44 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3223117847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:44.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:44 smithi029 ceph-mon[102350]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 17419/162174 objects degraded (10.741%); 1.3 MiB/s, 21 objects/s recovering 2024-03-29T04:47:44.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:44 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3223117847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:44.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:44 smithi029 ceph-mon[106624]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 17419/162174 objects degraded (10.741%); 1.3 MiB/s, 21 objects/s recovering 2024-03-29T04:47:44.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:44 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3223117847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:44.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 36 16 58227 58211 101.043 66.9375 0.0230947 0.00989052 2024-03-29T04:47:45.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:47:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:47:44] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-03-29T04:47:45.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 37 16 59007 58991 99.6299 48.75 0.0134508 0.0100314 2024-03-29T04:47:46.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:46 smithi099 ceph-mon[94036]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 17272/165513 objects degraded (10.435%); 2.1 MiB/s, 33 objects/s recovering 2024-03-29T04:47:46.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:46 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3426449200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:46.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:46 smithi029 ceph-mon[102350]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 17272/165513 objects degraded (10.435%); 2.1 MiB/s, 33 objects/s recovering 2024-03-29T04:47:46.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:46 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3426449200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:46.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:46 smithi029 ceph-mon[106624]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s; 17272/165513 objects degraded (10.435%); 2.1 MiB/s, 33 objects/s recovering 2024-03-29T04:47:46.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:46 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3426449200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:46.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 38 16 60021 60005 98.6758 63.375 0.00430569 0.0101285 2024-03-29T04:47:47.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:47 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 17272/165513 objects degraded (10.435%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:47.443 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:47 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 17272/165513 objects degraded (10.435%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:47.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:47 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 17272/165513 objects degraded (10.435%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:47.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 39 16 60955 60939 97.6424 58.375 0.022751 0.0102346 2024-03-29T04:47:48.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:48 smithi099 ceph-mon[94036]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s; 17272/172746 objects degraded (9.998%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:47:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:48 smithi029 ceph-mon[102350]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s; 17272/172746 objects degraded (9.998%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:47:48.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:48 smithi029 ceph-mon[106624]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s; 17272/172746 objects degraded (9.998%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:47:48.814 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:47:48.814268+0000 min lat: 0.000920881 max lat: 2.40197 avg lat: 0.0103249 2024-03-29T04:47:48.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:47:48.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 40 16 61813 61797 96.5419 53.625 0.0025822 0.0103249 2024-03-29T04:47:49.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:49 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3648528216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:49.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:49.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:49 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3648528216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:49.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:49.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:49 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3648528216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:49.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:49.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 41 16 62746 62730 95.6094 58.3125 0.00918372 0.0104535 2024-03-29T04:47:50.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:50 smithi099 ceph-mon[94036]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 17272/176307 objects degraded (9.797%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:47:50.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:50 smithi029 ceph-mon[102350]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 17272/176307 objects degraded (9.797%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:47:50.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:50 smithi029 ceph-mon[106624]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 17272/176307 objects degraded (9.797%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:47:50.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 42 16 63759 63743 94.8404 63.3125 0.0137781 0.0105384 2024-03-29T04:47:51.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:51 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/381629243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:51.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:51 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/381629243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:51.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:51 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/381629243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:51.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 43 16 64693 64677 93.9923 58.375 0.0122471 0.0106321 2024-03-29T04:47:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:52 smithi099 ceph-mon[94036]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 17108/185661 objects degraded (9.215%); 2.3 MiB/s, 37 objects/s recovering 2024-03-29T04:47:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:52 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 17108/185661 objects degraded (9.215%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[102350]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 17108/185661 objects degraded (9.215%); 2.3 MiB/s, 37 objects/s recovering 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 17108/185661 objects degraded (9.215%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[106624]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 17108/185661 objects degraded (9.215%); 2.3 MiB/s, 37 objects/s recovering 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:52.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:52 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 17108/185661 objects degraded (9.215%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:52.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 44 16 65664 65648 93.2353 60.6875 0.0100146 0.0107197 2024-03-29T04:47:53.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 45 16 66534 66518 92.3717 54.375 0.025679 0.0108182 2024-03-29T04:47:54.341 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[102350]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 17108/186564 objects degraded (9.170%); 1.6 MiB/s, 25 objects/s recovering 2024-03-29T04:47:54.341 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:54.341 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:54.341 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1643176585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:54.341 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[106624]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 17108/186564 objects degraded (9.170%); 1.6 MiB/s, 25 objects/s recovering 2024-03-29T04:47:54.342 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:54.342 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:54.342 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1643176585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:54.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:54 smithi099 ceph-mon[94036]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 17108/186564 objects degraded (9.170%); 1.6 MiB/s, 25 objects/s recovering 2024-03-29T04:47:54.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:54 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:54.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:54 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:47:54.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:54 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1643176585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:54.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 46 16 67448 67432 91.6054 57.125 0.00738814 0.010907 2024-03-29T04:47:54.944 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:47:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:47:54] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2024-03-29T04:47:55.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 47 16 68207 68191 90.6656 47.4375 0.00349366 0.0110222 2024-03-29T04:47:56.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:56 smithi099 ceph-mon[94036]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 16901/193713 objects degraded (8.725%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:47:56.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:56 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3293088461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:56.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:56 smithi029 ceph-mon[102350]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 16901/193713 objects degraded (8.725%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:47:56.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:56 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3293088461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:56.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:56 smithi029 ceph-mon[106624]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 55 MiB/s wr, 875 op/s; 16901/193713 objects degraded (8.725%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:47:56.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:56 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3293088461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:56.815 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 48 16 68874 68858 89.6452 41.6875 0.0110764 0.0111486 2024-03-29T04:47:57.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:57 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 16901/193713 objects degraded (8.725%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:57.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:57 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 16901/193713 objects degraded (8.725%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:57.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:57 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 16901/193713 objects degraded (8.725%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:47:57.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 49 16 69904 69888 89.1294 64.375 0.0095835 0.0112135 2024-03-29T04:47:58.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:58 smithi099 ceph-mon[94036]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 16901/200625 objects degraded (8.424%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:47:58.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:58 smithi029 ceph-mon[102350]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 16901/200625 objects degraded (8.424%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:47:58.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:58 smithi029 ceph-mon[106624]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s; 16901/200625 objects degraded (8.424%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:47:58.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 50 16 70879 70863 88.5655 60.9375 0.0166557 0.0112844 2024-03-29T04:47:59.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:59 smithi099 ceph-mon[94036]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 16901/203763 objects degraded (8.294%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:47:59.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:47:59 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3602040356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:59.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:59 smithi029 ceph-mon[102350]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 16901/203763 objects degraded (8.294%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:47:59.383 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:47:59 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3602040356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:59.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:59 smithi029 ceph-mon[106624]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 16901/203763 objects degraded (8.294%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:47:59.383 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:47:59 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3602040356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:47:59.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 51 16 71866 71850 88.0384 61.6875 0.00556728 0.0113531 2024-03-29T04:48:00.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 52 16 72802 72786 87.4703 58.5 0.0136166 0.011424 2024-03-29T04:48:01.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:01 smithi029 ceph-mon[102350]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 16681/212916 objects degraded (7.835%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:48:01.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/562545615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:01.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:01 smithi029 ceph-mon[106624]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 16681/212916 objects degraded (7.835%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:48:01.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/562545615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:01.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 53 16 73763 73747 86.9531 60.0625 0.00439405 0.0114928 2024-03-29T04:48:01.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:01 smithi099 ceph-mon[94036]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 16681/212916 objects degraded (7.835%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:48:01.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/562545615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:02.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:02 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 16681/212916 objects degraded (7.835%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:02.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:02 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 16681/212916 objects degraded (7.835%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:02.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 54 16 74617 74601 86.3312 53.375 0.0358763 0.0115779 2024-03-29T04:48:02.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:02 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 16681/212916 objects degraded (7.835%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:03.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:03 smithi029 ceph-mon[102350]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 16681/213759 objects degraded (7.804%); 2.2 MiB/s, 35 objects/s recovering 2024-03-29T04:48:03.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:03 smithi029 ceph-mon[106624]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 16681/213759 objects degraded (7.804%); 2.2 MiB/s, 35 objects/s recovering 2024-03-29T04:48:03.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 55 16 75438 75422 85.6944 51.3125 0.00953177 0.0116634 2024-03-29T04:48:03.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:03 smithi099 ceph-mon[94036]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 16681/213759 objects degraded (7.804%); 2.2 MiB/s, 35 objects/s recovering 2024-03-29T04:48:04.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:04 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4102316562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:04.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:04.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:04 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4102316562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:04.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:04.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 56 16 76306 76290 85.1329 54.25 0.0294858 0.0117398 2024-03-29T04:48:04.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:04 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4102316562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:04.874 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:05.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:48:04] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2024-03-29T04:48:05.628 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:05 smithi029 ceph-mon[102350]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 16681/217527 objects degraded (7.668%); 2.2 MiB/s, 35 objects/s recovering 2024-03-29T04:48:05.629 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:05 smithi029 ceph-mon[106624]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 16681/217527 objects degraded (7.668%); 2.2 MiB/s, 35 objects/s recovering 2024-03-29T04:48:05.775 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:05 smithi099 ceph-mon[94036]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 16681/217527 objects degraded (7.668%); 2.2 MiB/s, 35 objects/s recovering 2024-03-29T04:48:05.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 57 16 77183 77167 84.6009 54.8125 0.0172731 0.0118138 2024-03-29T04:48:06.290 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:05 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:48:05.955+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:06.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 58 16 78111 78095 84.1422 58 0.0374096 0.0118785 2024-03-29T04:48:07.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 16436/226671 objects degraded (7.251%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:07.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2683118252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:07.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:06 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 16681/217527 objects degraded (7.668%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 16436/226671 objects degraded (7.251%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2683118252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 16681/217527 objects degraded (7.668%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:07.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s; 16436/226671 objects degraded (7.251%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2683118252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:07.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:06 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 16681/217527 objects degraded (7.668%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:07.816 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 59 16 79111 79095 83.7753 62.5 0.0403722 0.0119302 2024-03-29T04:48:08.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:48:08.816091+0000 min lat: 0.000920881 max lat: 2.40197 avg lat: 0.0119849 2024-03-29T04:48:08.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:48:08.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 60 16 80082 80066 83.3904 60.6875 0.0140116 0.0119849 2024-03-29T04:48:09.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:08 smithi099 ceph-mon[94036]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 16436/230646 objects degraded (7.126%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:48:09.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:08 smithi029 ceph-mon[102350]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 16436/230646 objects degraded (7.126%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:48:09.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:08 smithi029 ceph-mon[106624]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 16436/230646 objects degraded (7.126%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:48:09.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 61 16 81096 81080 83.0622 63.375 0.00877055 0.0120329 2024-03-29T04:48:10.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2750004794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:10.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2750004794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:10.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2750004794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:10.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 62 16 82059 82043 82.6932 60.1875 0.0123278 0.0120863 2024-03-29T04:48:11.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:10 smithi099 ceph-mon[94036]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 16179/234240 objects degraded (6.907%); 4.0 MiB/s, 63 objects/s recovering 2024-03-29T04:48:11.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:10 smithi029 ceph-mon[102350]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 16179/234240 objects degraded (6.907%); 4.0 MiB/s, 63 objects/s recovering 2024-03-29T04:48:11.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:10 smithi029 ceph-mon[106624]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 16179/234240 objects degraded (6.907%); 4.0 MiB/s, 63 objects/s recovering 2024-03-29T04:48:11.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 63 16 82983 82967 82.2972 57.75 0.00626646 0.0121444 2024-03-29T04:48:12.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:11 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 16179/234240 objects degraded (6.907%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:12.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:11 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3534906229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:12.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:11 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 16179/234240 objects degraded (6.907%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:12.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:11 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3534906229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:12.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:11 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 16179/234240 objects degraded (6.907%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:12.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:11 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3534906229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:12.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 64 16 83977 83961 81.9819 62.125 0.00734939 0.0121917 2024-03-29T04:48:13.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:12 smithi099 ceph-mon[94036]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 16179/241590 objects degraded (6.697%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:13.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:12 smithi029 ceph-mon[102350]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 16179/241590 objects degraded (6.697%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:13.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:12 smithi029 ceph-mon[106624]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 16179/241590 objects degraded (6.697%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:13.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 65 16 84977 84961 81.6821 62.5 0.00809487 0.0122359 2024-03-29T04:48:14.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 66 16 85891 85875 81.31 57.125 0.0295942 0.0122926 2024-03-29T04:48:15.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:14 smithi099 ceph-mon[94036]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 16179/245259 objects degraded (6.597%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:15.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:14 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1752925004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:15.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:14 smithi029 ceph-mon[102350]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 16179/245259 objects degraded (6.597%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:15.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:14 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1752925004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:15.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:48:14] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2024-03-29T04:48:15.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:14 smithi029 ceph-mon[106624]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 16179/245259 objects degraded (6.597%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:15.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:14 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1752925004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:15.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 67 16 86886 86870 81.0245 62.1875 0.00673888 0.0123337 2024-03-29T04:48:16.817 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 68 16 87831 87815 80.7015 59.0625 0.00866233 0.0123834 2024-03-29T04:48:17.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:16 smithi099 ceph-mon[94036]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 15882/255138 objects degraded (6.225%); 4.4 MiB/s, 70 objects/s recovering 2024-03-29T04:48:17.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:16 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 15882/255138 objects degraded (6.225%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:17.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:16 smithi029 ceph-mon[102350]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 15882/255138 objects degraded (6.225%); 4.4 MiB/s, 70 objects/s recovering 2024-03-29T04:48:17.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:16 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 15882/255138 objects degraded (6.225%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:17.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:16 smithi029 ceph-mon[106624]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s; 15882/255138 objects degraded (6.225%); 4.4 MiB/s, 70 objects/s recovering 2024-03-29T04:48:17.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:16 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 15882/255138 objects degraded (6.225%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:17.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 69 16 88798 88782 80.4077 60.4375 0.00575881 0.0124305 2024-03-29T04:48:18.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:17 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1183898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:18.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:17 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1183898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:18.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:17 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1183898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:18.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 70 16 89762 89746 80.1197 60.25 0.0106751 0.0124751 2024-03-29T04:48:19.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:18 smithi099 ceph-mon[94036]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 15882/259704 objects degraded (6.115%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:48:19.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:18 smithi029 ceph-mon[102350]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 15882/259704 objects degraded (6.115%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:48:19.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:18 smithi029 ceph-mon[106624]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 15882/259704 objects degraded (6.115%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:48:19.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 71 16 90715 90699 79.8301 59.5625 0.0108835 0.0125204 2024-03-29T04:48:20.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:20.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:19 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3061656843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:20.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:20.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:19 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3061656843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:20.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:20.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:19 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3061656843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:20.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 72 16 91658 91642 79.5399 58.9375 0.0065405 0.0125662 2024-03-29T04:48:21.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:20 smithi029 ceph-mon[102350]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 15533/263376 objects degraded (5.898%); 4.7 MiB/s, 75 objects/s recovering 2024-03-29T04:48:21.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:48:20.972+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:21.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:20 smithi029 ceph-mon[106624]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 15533/263376 objects degraded (5.898%); 4.7 MiB/s, 75 objects/s recovering 2024-03-29T04:48:21.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:20 smithi099 ceph-mon[94036]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 15533/263376 objects degraded (5.898%); 4.7 MiB/s, 75 objects/s recovering 2024-03-29T04:48:21.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 73 16 92595 92579 79.2524 58.5625 0.0168249 0.0126102 2024-03-29T04:48:22.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 15533/263376 objects degraded (5.898%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:22.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:21 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 15533/263376 objects degraded (5.898%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:22.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:22.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:21 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:22.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:21 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:48:22.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:21 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 15533/263376 objects degraded (5.898%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:22.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 74 16 93522 93506 78.9643 57.9375 0.0351174 0.0126579 2024-03-29T04:48:23.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:22 smithi029 ceph-mon[102350]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 15533/270597 objects degraded (5.740%); 3.4 MiB/s, 53 objects/s recovering 2024-03-29T04:48:23.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:22 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2334404882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:23.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:22 smithi029 ceph-mon[106624]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 15533/270597 objects degraded (5.740%); 3.4 MiB/s, 53 objects/s recovering 2024-03-29T04:48:23.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:22 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2334404882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:23.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:22 smithi099 ceph-mon[94036]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 15533/270597 objects degraded (5.740%); 3.4 MiB/s, 53 objects/s recovering 2024-03-29T04:48:23.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:22 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2334404882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:23.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 75 16 94439 94423 78.6756 57.3125 0.0307371 0.0127032 2024-03-29T04:48:24.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 76 16 94785 94769 77.9249 21.625 0.00331433 0.0127065 2024-03-29T04:48:25.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:48:24] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 15533/274101 objects degraded (5.667%); 3.4 MiB/s, 53 objects/s recovering 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: osdmap e113: 8 total, 8 up, 8 in 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1263477405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 15533/274101 objects degraded (5.667%); 3.4 MiB/s, 53 objects/s recovering 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: osdmap e113: 8 total, 8 up, 8 in 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:24 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1263477405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s; 15533/274101 objects degraded (5.667%); 3.4 MiB/s, 53 objects/s recovering 2024-03-29T04:48:25.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: osdmap e113: 8 total, 8 up, 8 in 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: failed to encode map e113 with expected crc 2024-03-29T04:48:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:24 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1263477405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:25.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 77 16 95263 95247 77.3009 29.875 0.00362357 0.0129087 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[102350]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[102350]: osdmap e114: 8 total, 8 up, 8 in 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[102350]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[102350]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[102350]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[106624]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.116 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[106624]: osdmap e114: 8 total, 8 up, 8 in 2024-03-29T04:48:26.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[106624]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[106624]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.117 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:25 smithi029 ceph-mon[106624]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:25 smithi099 ceph-mon[94036]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:25 smithi099 ceph-mon[94036]: osdmap e114: 8 total, 8 up, 8 in 2024-03-29T04:48:26.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:25 smithi099 ceph-mon[94036]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:25 smithi099 ceph-mon[94036]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:25 smithi099 ceph-mon[94036]: failed to encode map e114 with expected crc 2024-03-29T04:48:26.818 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 78 16 95925 95909 76.8403 41.375 0.0216263 0.0130068 2024-03-29T04:48:27.365 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:26 smithi029 ceph-mon[102350]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 15533/281742 objects degraded (5.513%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:27.365 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:26 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 15533/281742 objects degraded (5.513%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:27.365 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:26 smithi029 ceph-mon[106624]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 15533/281742 objects degraded (5.513%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:27.365 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:26 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 15533/281742 objects degraded (5.513%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:27.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:26 smithi099 ceph-mon[94036]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 15533/281742 objects degraded (5.513%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:48:27.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:26 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 15533/281742 objects degraded (5.513%), 7 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:27.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 79 16 96757 96741 76.5258 52 0.0192327 0.0130617 2024-03-29T04:48:28.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:27 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2952764914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:28.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:27 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2952764914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:28.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:27 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2952764914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:28.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:2024-03-29T04:48:28.817920+0000 min lat: 0.000920881 max lat: 2.40197 avg lat: 0.013103 2024-03-29T04:48:28.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-03-29T04:48:28.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 80 15 97671 97656 76.284 57.1875 0.0108889 0.013103 2024-03-29T04:48:29.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:28 smithi099 ceph-mon[94036]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 15346/284979 objects degraded (5.385%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:48:29.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:28 smithi029 ceph-mon[102350]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 15346/284979 objects degraded (5.385%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:48:29.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:28 smithi029 ceph-mon[106624]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 15346/284979 objects degraded (5.385%); 1.5 MiB/s, 23 objects/s recovering 2024-03-29T04:48:29.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 81 16 98505 98489 75.9849 52.0625 0.0291258 0.0131545 2024-03-29T04:48:30.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:29 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3818250519' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:30.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:29 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3818250519' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:30.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:29 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3818250519' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:30.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 82 16 99360 99344 75.7099 53.4375 0.0194775 0.0132002 2024-03-29T04:48:31.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:30 smithi099 ceph-mon[94036]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 15013/288036 objects degraded (5.212%); 4.1 MiB/s, 64 objects/s recovering 2024-03-29T04:48:31.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:30 smithi029 ceph-mon[102350]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 15013/288036 objects degraded (5.212%); 4.1 MiB/s, 64 objects/s recovering 2024-03-29T04:48:31.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:30 smithi029 ceph-mon[106624]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 15013/288036 objects degraded (5.212%); 4.1 MiB/s, 64 objects/s recovering 2024-03-29T04:48:31.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 83 16 100280 100264 75.4904 57.5 0.0398018 0.01324 2024-03-29T04:48:32.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:31 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 15013/288036 objects degraded (5.212%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:32.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:31 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 15013/288036 objects degraded (5.212%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:32.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:31 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 15013/288036 objects degraded (5.212%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:32.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 84 16 101129 101113 75.2234 53.0625 0.0160243 0.0132874 2024-03-29T04:48:33.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:32 smithi099 ceph-mon[94036]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 14819/293808 objects degraded (5.044%); 5.6 MiB/s, 89 objects/s recovering 2024-03-29T04:48:33.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:32 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2600934707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:33.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:32 smithi029 ceph-mon[102350]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 14819/293808 objects degraded (5.044%); 5.6 MiB/s, 89 objects/s recovering 2024-03-29T04:48:33.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:32 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2600934707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:33.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:32 smithi029 ceph-mon[106624]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 14819/293808 objects degraded (5.044%); 5.6 MiB/s, 89 objects/s recovering 2024-03-29T04:48:33.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:32 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2600934707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:33.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 85 16 102057 102041 75.0207 58 0.00858118 0.0133234 2024-03-29T04:48:34.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:33 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:34.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:33 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:34.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:33 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:34.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 86 16 102892 102876 74.7552 52.1875 0.020135 0.0133702 2024-03-29T04:48:35.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:35 smithi029 ceph-mon[102350]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 14819/298029 objects degraded (4.972%); 4.5 MiB/s, 72 objects/s recovering 2024-03-29T04:48:35.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:35 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1367588355' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:35.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:48:34] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2024-03-29T04:48:35.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:35 smithi029 ceph-mon[106624]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 14819/298029 objects degraded (4.972%); 4.5 MiB/s, 72 objects/s recovering 2024-03-29T04:48:35.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:35 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1367588355' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:35.275 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:34 smithi099 ceph-mon[94036]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 14819/298029 objects degraded (4.972%); 4.5 MiB/s, 72 objects/s recovering 2024-03-29T04:48:35.275 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:35 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1367588355' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:35.819 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 87 16 103731 103715 74.4986 52.4375 0.0121078 0.0134174 2024-03-29T04:48:36.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:36 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:36.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:36 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:36.445 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:35 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:48:35.988+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:36.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:36 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:36.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 88 16 104613 104597 74.2784 55.125 0.0218788 0.0134572 2024-03-29T04:48:37.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:37 smithi099 ceph-mon[94036]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 14819/306261 objects degraded (4.839%); 4.1 MiB/s, 65 objects/s recovering 2024-03-29T04:48:37.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:37 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:37.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:37 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:37.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:37 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 14819/306261 objects degraded (4.839%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[102350]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 14819/306261 objects degraded (4.839%); 4.1 MiB/s, 65 objects/s recovering 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 14819/306261 objects degraded (4.839%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[106624]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 14819/306261 objects degraded (4.839%); 4.1 MiB/s, 65 objects/s recovering 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:37.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:37 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 14819/306261 objects degraded (4.839%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:37.820 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 89 16 105421 105405 74.0112 50.5 0.0236241 0.0135054 2024-03-29T04:48:38.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4221876471' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:38.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4221876471' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:38.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4221876471' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:38.820 INFO:tasks.radosbench.radosbench.0.smithi029.stdout: 90 6 106259 106253 73.7777 53 0.0519248 0.0135476 2024-03-29T04:48:39.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:39 smithi099 ceph-mon[94036]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 14618/310326 objects degraded (4.711%); 4.8 MiB/s, 76 objects/s recovering 2024-03-29T04:48:39.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:39 smithi029 ceph-mon[102350]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 14618/310326 objects degraded (4.711%); 4.8 MiB/s, 76 objects/s recovering 2024-03-29T04:48:39.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:39 smithi029 ceph-mon[106624]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s; 14618/310326 objects degraded (4.711%); 4.8 MiB/s, 76 objects/s recovering 2024-03-29T04:48:39.820 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total time run: 90.0142 2024-03-29T04:48:39.820 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Total writes made: 106259 2024-03-29T04:48:39.820 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Write size: 65536 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Object size: 65536 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Bandwidth (MB/sec): 73.7794 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Bandwidth: 47.1601 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max bandwidth (MB/sec): 252.125 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min bandwidth (MB/sec): 0 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average IOPS: 1180 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev IOPS: 754.568 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max IOPS: 4034 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min IOPS: 0 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Average Latency(s): 0.0135486 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Stddev Latency(s): 0.0355785 2024-03-29T04:48:39.821 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Max latency(s): 2.40197 2024-03-29T04:48:39.822 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Min latency(s): 0.000920881 2024-03-29T04:48:39.825 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Cleaning up (deleting benchmark objects) 2024-03-29T04:48:41.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:41 smithi099 ceph-mon[94036]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 14618/313164 objects degraded (4.668%); 3.8 MiB/s, 60 objects/s recovering 2024-03-29T04:48:41.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:41 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3427165426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:41.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:41 smithi029 ceph-mon[102350]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 14618/313164 objects degraded (4.668%); 3.8 MiB/s, 60 objects/s recovering 2024-03-29T04:48:41.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:41 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3427165426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:41.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:41 smithi029 ceph-mon[106624]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s; 14618/313164 objects degraded (4.668%); 3.8 MiB/s, 60 objects/s recovering 2024-03-29T04:48:41.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:41 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3427165426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:42.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:42 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 14618/313164 objects degraded (4.668%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:42.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:42 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 14618/313164 objects degraded (4.668%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:42.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:42 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 14618/313164 objects degraded (4.668%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:43.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:43 smithi099 ceph-mon[94036]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 838 op/s; 14252/317463 objects degraded (4.489%); 4.0 MiB/s, 63 objects/s recovering 2024-03-29T04:48:43.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:43 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3746745071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:43.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:43 smithi029 ceph-mon[102350]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 838 op/s; 14252/317463 objects degraded (4.489%); 4.0 MiB/s, 63 objects/s recovering 2024-03-29T04:48:43.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:43 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3746745071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:43.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:43 smithi029 ceph-mon[106624]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 838 op/s; 14252/317463 objects degraded (4.489%); 4.0 MiB/s, 63 objects/s recovering 2024-03-29T04:48:43.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:43 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3746745071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:45.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:45 smithi029 ceph-mon[102350]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 770 op/s; 14252/316140 objects degraded (4.508%); 3.0 MiB/s, 47 objects/s recovering 2024-03-29T04:48:45.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:45 smithi029 ceph-mon[106624]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 770 op/s; 14252/316140 objects degraded (4.508%); 3.0 MiB/s, 47 objects/s recovering 2024-03-29T04:48:45.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:48:44] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.43.0" 2024-03-29T04:48:45.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:45 smithi099 ceph-mon[94036]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 770 op/s; 14252/316140 objects degraded (4.508%); 3.0 MiB/s, 47 objects/s recovering 2024-03-29T04:48:46.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:46 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2263121909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:46.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:46 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2263121909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:46.444 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:46 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2263121909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:47.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:47 smithi099 ceph-mon[94036]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 880 op/s; 14252/307974 objects degraded (4.628%); 3.0 MiB/s, 47 objects/s recovering 2024-03-29T04:48:47.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:47 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 14252/307974 objects degraded (4.628%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:47.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:47 smithi029 ceph-mon[102350]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 880 op/s; 14252/307974 objects degraded (4.628%); 3.0 MiB/s, 47 objects/s recovering 2024-03-29T04:48:47.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:47 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 14252/307974 objects degraded (4.628%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:47.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:47 smithi029 ceph-mon[106624]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 880 op/s; 14252/307974 objects degraded (4.628%); 3.0 MiB/s, 47 objects/s recovering 2024-03-29T04:48:47.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:47 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 14252/307974 objects degraded (4.628%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:48.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:48 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3119882833' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:48.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:48 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3119882833' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:48.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:48 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3119882833' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:49.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:49 smithi099 ceph-mon[94036]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 765 op/s; 14020/303858 objects degraded (4.614%); 4.2 MiB/s, 66 objects/s recovering 2024-03-29T04:48:49.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:49.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:49 smithi029 ceph-mon[102350]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 765 op/s; 14020/303858 objects degraded (4.614%); 4.2 MiB/s, 66 objects/s recovering 2024-03-29T04:48:49.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:49.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:49 smithi029 ceph-mon[106624]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 765 op/s; 14020/303858 objects degraded (4.614%); 4.2 MiB/s, 66 objects/s recovering 2024-03-29T04:48:49.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:51.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:51 smithi099 ceph-mon[94036]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 743 op/s; 14020/300591 objects degraded (4.664%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:48:51.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:51 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/802031544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:51.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:51 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:51.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-mon[102350]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 743 op/s; 14020/300591 objects degraded (4.664%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:48:51.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/802031544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:51.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:51.445 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:48:51.007+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:51.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-mon[106624]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 15 MiB/s wr, 743 op/s; 14020/300591 objects degraded (4.664%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:48:51.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/802031544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:51.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:51 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:52.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:52 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:52.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:52 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:52.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:52 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 14020/300591 objects degraded (4.664%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:52.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:52 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:52.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:52 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:52.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:52 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 14020/300591 objects degraded (4.664%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:52.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:52 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:48:52.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:52 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:48:52.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:52 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 14020/300591 objects degraded (4.664%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:53.374 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:53 smithi099 ceph-mon[94036]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 840 op/s; 13791/294270 objects degraded (4.687%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:48:53.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:53 smithi029 ceph-mon[102350]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 840 op/s; 13791/294270 objects degraded (4.687%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:48:53.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:53 smithi029 ceph-mon[106624]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 840 op/s; 13791/294270 objects degraded (4.687%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:48:54.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:54 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/889768640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:54.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:54 smithi099 ceph-mon[94036]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.7 MiB/s wr, 812 op/s; 13791/290226 objects degraded (4.752%); 2.4 MiB/s, 38 objects/s recovering 2024-03-29T04:48:54.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:54 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/889768640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:54.444 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:54 smithi029 ceph-mon[102350]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.7 MiB/s wr, 812 op/s; 13791/290226 objects degraded (4.752%); 2.4 MiB/s, 38 objects/s recovering 2024-03-29T04:48:54.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:54 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/889768640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:54.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:54 smithi029 ceph-mon[106624]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.7 MiB/s wr, 812 op/s; 13791/290226 objects degraded (4.752%); 2.4 MiB/s, 38 objects/s recovering 2024-03-29T04:48:55.194 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:48:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:48:54] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2024-03-29T04:48:56.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:55 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3712601710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:56.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:55 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3712601710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:56.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:55 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3712601710' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:57.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:56 smithi099 ceph-mon[94036]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 13791/282099 objects degraded (4.889%); 2.4 MiB/s, 38 objects/s recovering 2024-03-29T04:48:57.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:56 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 13791/282099 objects degraded (4.889%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:57.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:56 smithi029 ceph-mon[102350]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 13791/282099 objects degraded (4.889%); 2.4 MiB/s, 38 objects/s recovering 2024-03-29T04:48:57.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:56 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 13791/282099 objects degraded (4.889%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:57.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:56 smithi029 ceph-mon[106624]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 13791/282099 objects degraded (4.889%); 2.4 MiB/s, 38 objects/s recovering 2024-03-29T04:48:57.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:56 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 13791/282099 objects degraded (4.889%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:48:59.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:58 smithi099 ceph-mon[94036]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 13532/278205 objects degraded (4.864%); 3.7 MiB/s, 59 objects/s recovering 2024-03-29T04:48:59.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:48:58 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3474575512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:59.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:58 smithi029 ceph-mon[102350]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 13532/278205 objects degraded (4.864%); 3.7 MiB/s, 59 objects/s recovering 2024-03-29T04:48:59.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:48:58 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3474575512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:48:59.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:58 smithi029 ceph-mon[106624]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 13532/278205 objects degraded (4.864%); 3.7 MiB/s, 59 objects/s recovering 2024-03-29T04:48:59.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:48:58 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3474575512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:01.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:00 smithi099 ceph-mon[94036]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 13532/275040 objects degraded (4.920%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:49:01.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:00 smithi029 ceph-mon[102350]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 13532/275040 objects degraded (4.920%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:49:01.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:00 smithi029 ceph-mon[106624]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 13532/275040 objects degraded (4.920%); 2.5 MiB/s, 40 objects/s recovering 2024-03-29T04:49:02.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2397161770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:02.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:01 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 13532/275040 objects degraded (4.920%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:02.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2397161770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:01 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 13532/275040 objects degraded (4.920%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:02.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2397161770' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:02.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:01 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 13532/275040 objects degraded (4.920%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:03.124 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:02 smithi099 ceph-mon[94036]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 13238/268815 objects degraded (4.925%); 4.1 MiB/s, 65 objects/s recovering 2024-03-29T04:49:03.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:02 smithi029 ceph-mon[102350]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 13238/268815 objects degraded (4.925%); 4.1 MiB/s, 65 objects/s recovering 2024-03-29T04:49:03.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:02 smithi029 ceph-mon[106624]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 13238/268815 objects degraded (4.925%); 4.1 MiB/s, 65 objects/s recovering 2024-03-29T04:49:04.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1813754408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:04.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1813754408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:04.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1813754408' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:05.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:04 smithi099 ceph-mon[94036]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 13238/265197 objects degraded (4.992%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:05.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:05.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:04 smithi029 ceph-mon[102350]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 13238/265197 objects degraded (4.992%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:05.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:05.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:49:04] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2024-03-29T04:49:05.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:04 smithi029 ceph-mon[106624]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 13238/265197 objects degraded (4.992%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:05.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:06 smithi099 ceph-mon[94036]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 13238/257673 objects degraded (5.138%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2963588525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:07.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:06 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 13238/257673 objects degraded (5.138%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:07.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[106624]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 13238/257673 objects degraded (5.138%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:07.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:07.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2963588525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:07.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 13238/257673 objects degraded (5.138%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[102350]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 13238/257673 objects degraded (5.138%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:07.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:07.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2963588525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:07.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:06 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 13238/257673 objects degraded (5.138%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:09.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:08 smithi099 ceph-mon[94036]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 12934/253938 objects degraded (5.093%); 4.5 MiB/s, 71 objects/s recovering 2024-03-29T04:49:09.125 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:08 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1368555176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:09.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:08 smithi029 ceph-mon[102350]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 12934/253938 objects degraded (5.093%); 4.5 MiB/s, 71 objects/s recovering 2024-03-29T04:49:09.126 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:08 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1368555176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:09.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:08 smithi029 ceph-mon[106624]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 12934/253938 objects degraded (5.093%); 4.5 MiB/s, 71 objects/s recovering 2024-03-29T04:49:09.126 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:08 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1368555176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:11.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 12934/251085 objects degraded (5.151%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:49:11.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: osdmap e115: 8 total, 8 up, 8 in 2024-03-29T04:49:11.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.036 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:10 smithi099 ceph-mon[94036]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 12934/251085 objects degraded (5.151%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:49:11.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: osdmap e115: 8 total, 8 up, 8 in 2024-03-29T04:49:11.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[102350]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 12934/251085 objects degraded (5.151%); 3.1 MiB/s, 49 objects/s recovering 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: osdmap e115: 8 total, 8 up, 8 in 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:10 smithi029 ceph-mon[106624]: failed to encode map e115 with expected crc 2024-03-29T04:49:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: osdmap e116: 8 total, 8 up, 8 in 2024-03-29T04:49:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1024905173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 12934/251085 objects degraded (5.151%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: osdmap e116: 8 total, 8 up, 8 in 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: failed to encode map e116 with expected crc 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1024905173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:11.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:11.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:11.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:11 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 12934/251085 objects degraded (5.151%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: failed to encode map e116 with expected crc 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: failed to encode map e116 with expected crc 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: osdmap e116: 8 total, 8 up, 8 in 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: failed to encode map e116 with expected crc 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: failed to encode map e116 with expected crc 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: failed to encode map e116 with expected crc 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1024905173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:12.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:12.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:12.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:11 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 12934/251085 objects degraded (5.151%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:13.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:13 smithi099 ceph-mon[94036]: pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 12891/246348 objects degraded (5.233%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:49:13.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:13 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:13.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:13 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:13.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-mon[102350]: pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 12891/246348 objects degraded (5.233%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:49:13.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:13.376 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:13.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-mon[106624]: pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 12891/246348 objects degraded (5.233%); 2.7 MiB/s, 43 objects/s recovering 2024-03-29T04:49:13.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:13.376 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:13.694 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:13 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:49:13.373+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:49:14.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:14.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:14.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:14.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:14.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:14.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:14 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/505919216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:14.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/505919216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:14.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:14.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:14.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:49:14.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:14.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:14.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:14.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/505919216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:15.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:49:14] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.43.0" 2024-03-29T04:49:15.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:15 smithi029 ceph-mon[102350]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 12891/243813 objects degraded (5.287%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:15.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:15 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:15.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:15 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:49:15.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:15 smithi029 ceph-mon[106624]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 12891/243813 objects degraded (5.287%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:15.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:15 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:15.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:15 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:49:15.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:15 smithi099 ceph-mon[94036]: pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 12891/243813 objects degraded (5.287%); 2.9 MiB/s, 46 objects/s recovering 2024-03-29T04:49:15.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:15 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:15.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:15 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-29T04:49:17.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:17 smithi099 ceph-mon[94036]: pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12430/236253 objects degraded (5.261%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:17.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:17 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/707154747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:17.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:17 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 12430/236253 objects degraded (5.261%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:17.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:17 smithi029 ceph-mon[102350]: pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12430/236253 objects degraded (5.261%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:17.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:17 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/707154747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:17.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:17 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 12430/236253 objects degraded (5.261%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:17.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:17 smithi029 ceph-mon[106624]: pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 12430/236253 objects degraded (5.261%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:17.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:17 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/707154747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:17.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:17 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 12430/236253 objects degraded (5.261%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:19.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:19 smithi099 ceph-mon[94036]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 12430/236253 objects degraded (5.261%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:19.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:19.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:19 smithi029 ceph-mon[102350]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 12430/236253 objects degraded (5.261%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:19.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:19.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:19 smithi029 ceph-mon[106624]: pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 12430/236253 objects degraded (5.261%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:19.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:20.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:20 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/138729805' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:20.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:20 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/138729805' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:20.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:20 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/138729805' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:21.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:21 smithi099 ceph-mon[94036]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 12430/231630 objects degraded (5.366%); 3.4 MiB/s, 54 objects/s recovering 2024-03-29T04:49:21.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:21 smithi029 ceph-mon[102350]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 12430/231630 objects degraded (5.366%); 3.4 MiB/s, 54 objects/s recovering 2024-03-29T04:49:21.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:21 smithi029 ceph-mon[106624]: pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 12430/231630 objects degraded (5.366%); 3.4 MiB/s, 54 objects/s recovering 2024-03-29T04:49:22.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:22 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 12430/231630 objects degraded (5.366%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:22.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:22 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/467635900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:22.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:22 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 12430/231630 objects degraded (5.366%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:22.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:22 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/467635900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:22.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:22 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 12430/231630 objects degraded (5.366%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:22.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:22 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/467635900' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:23.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:23 smithi099 ceph-mon[94036]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 12243/223986 objects degraded (5.466%); 3.9 MiB/s, 63 objects/s recovering 2024-03-29T04:49:23.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:23 smithi029 ceph-mon[102350]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 12243/223986 objects degraded (5.466%); 3.9 MiB/s, 63 objects/s recovering 2024-03-29T04:49:23.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:23 smithi029 ceph-mon[106624]: pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 12243/223986 objects degraded (5.466%); 3.9 MiB/s, 63 objects/s recovering 2024-03-29T04:49:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:25 smithi029 ceph-mon[102350]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 12243/220800 objects degraded (5.545%); 3.6 MiB/s, 56 objects/s recovering 2024-03-29T04:49:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1001661899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:25.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:49:24] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2024-03-29T04:49:25.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:25 smithi029 ceph-mon[106624]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 12243/220800 objects degraded (5.545%); 3.6 MiB/s, 56 objects/s recovering 2024-03-29T04:49:25.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:25 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1001661899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:25 smithi099 ceph-mon[94036]: pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 12243/220800 objects degraded (5.545%); 3.6 MiB/s, 56 objects/s recovering 2024-03-29T04:49:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:25 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1001661899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:27.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:27 smithi099 ceph-mon[94036]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 12066/211800 objects degraded (5.697%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:27.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:27 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 12066/211800 objects degraded (5.697%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:27.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:27 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3054300357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:27.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:27 smithi029 ceph-mon[102350]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 12066/211800 objects degraded (5.697%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:27.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:27 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 12066/211800 objects degraded (5.697%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:27.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:27 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3054300357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:27.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:27 smithi029 ceph-mon[106624]: pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 888 op/s; 12066/211800 objects degraded (5.697%); 4.3 MiB/s, 68 objects/s recovering 2024-03-29T04:49:27.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:27 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 12066/211800 objects degraded (5.697%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:27.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:27 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3054300357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:28.695 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:28 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:49:28.392+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:29.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:29 smithi099 ceph-mon[94036]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 12066/211800 objects degraded (5.697%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:49:29.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:29 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:29.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:29 smithi029 ceph-mon[102350]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 12066/211800 objects degraded (5.697%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:49:29.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:29 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:29.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:29 smithi029 ceph-mon[106624]: pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 12066/211800 objects degraded (5.697%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:49:29.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:29 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:30.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:30 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:30.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:30 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:30.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:30 smithi099 ceph-mon[94036]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12066/207378 objects degraded (5.818%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:49:30.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:30 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1380514107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[102350]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12066/207378 objects degraded (5.818%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1380514107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:30.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[106624]: pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 12066/207378 objects degraded (5.818%); 1.9 MiB/s, 30 objects/s recovering 2024-03-29T04:49:30.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:30 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1380514107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:32.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:32 smithi099 ceph-mon[94036]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 11894/200133 objects degraded (5.943%); 2.8 MiB/s, 44 objects/s recovering 2024-03-29T04:49:32.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:32 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11894/200133 objects degraded (5.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:32.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:32 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1484230164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:32.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:32 smithi029 ceph-mon[102350]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 11894/200133 objects degraded (5.943%); 2.8 MiB/s, 44 objects/s recovering 2024-03-29T04:49:32.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:32 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11894/200133 objects degraded (5.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:32.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:32 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1484230164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:32.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:32 smithi029 ceph-mon[106624]: pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 11894/200133 objects degraded (5.943%); 2.8 MiB/s, 44 objects/s recovering 2024-03-29T04:49:32.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:32 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11894/200133 objects degraded (5.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:32.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:32 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1484230164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:34.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:34 smithi099 ceph-mon[94036]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 11894/197163 objects degraded (6.033%); 1.8 MiB/s, 29 objects/s recovering 2024-03-29T04:49:34.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:34.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:34 smithi029 ceph-mon[102350]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 11894/197163 objects degraded (6.033%); 1.8 MiB/s, 29 objects/s recovering 2024-03-29T04:49:34.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:34.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:34 smithi029 ceph-mon[106624]: pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 11894/197163 objects degraded (6.033%); 1.8 MiB/s, 29 objects/s recovering 2024-03-29T04:49:34.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:35.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:49:34] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2024-03-29T04:49:35.526 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:35 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1278161714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:35.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:35 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1278161714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:35.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:35 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1278161714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:36.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:36 smithi099 ceph-mon[94036]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 11702/188040 objects degraded (6.223%); 1.8 MiB/s, 45 objects/s recovering 2024-03-29T04:49:36.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:36 smithi029 ceph-mon[102350]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 11702/188040 objects degraded (6.223%); 1.8 MiB/s, 45 objects/s recovering 2024-03-29T04:49:36.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:36 smithi029 ceph-mon[106624]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 11702/188040 objects degraded (6.223%); 1.8 MiB/s, 45 objects/s recovering 2024-03-29T04:49:37.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:37 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11702/188040 objects degraded (6.223%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:37.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:37 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11702/188040 objects degraded (6.223%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:37.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:37 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11702/188040 objects degraded (6.223%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:38.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:38 smithi099 ceph-mon[94036]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 11702/188040 objects degraded (6.223%); 901 KiB/s, 30 objects/s recovering 2024-03-29T04:49:38.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/615527462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:38.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:38 smithi029 ceph-mon[102350]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 11702/188040 objects degraded (6.223%); 901 KiB/s, 30 objects/s recovering 2024-03-29T04:49:38.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/615527462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:38.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:38 smithi029 ceph-mon[106624]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 11702/188040 objects degraded (6.223%); 901 KiB/s, 30 objects/s recovering 2024-03-29T04:49:38.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/615527462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:40.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:40 smithi099 ceph-mon[94036]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11702/183783 objects degraded (6.367%); 901 KiB/s, 30 objects/s recovering 2024-03-29T04:49:40.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:40 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2226448230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:40.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:40 smithi029 ceph-mon[102350]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11702/183783 objects degraded (6.367%); 901 KiB/s, 30 objects/s recovering 2024-03-29T04:49:40.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:40 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2226448230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:40.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:40 smithi029 ceph-mon[106624]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11702/183783 objects degraded (6.367%); 901 KiB/s, 30 objects/s recovering 2024-03-29T04:49:40.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:40 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2226448230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:42.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:42 smithi099 ceph-mon[94036]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 11499/176172 objects degraded (6.527%); 901 KiB/s, 47 objects/s recovering 2024-03-29T04:49:42.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:42 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11499/176172 objects degraded (6.527%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:42.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:42 smithi029 ceph-mon[102350]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 11499/176172 objects degraded (6.527%); 901 KiB/s, 47 objects/s recovering 2024-03-29T04:49:42.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:42 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11499/176172 objects degraded (6.527%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:42.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:42 smithi029 ceph-mon[106624]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 11499/176172 objects degraded (6.527%); 901 KiB/s, 47 objects/s recovering 2024-03-29T04:49:42.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:42 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11499/176172 objects degraded (6.527%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:43.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:43 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2091094660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:43.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:43 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2091094660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:43.695 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:43 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:49:43.410+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:43.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:43 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2091094660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:44.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:44 smithi099 ceph-mon[94036]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11499/173103 objects degraded (6.643%); 0 B/s, 32 objects/s recovering 2024-03-29T04:49:44.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:44 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:44.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:44 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:44.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:44 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:44.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[102350]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11499/173103 objects degraded (6.643%); 0 B/s, 32 objects/s recovering 2024-03-29T04:49:44.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:44.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:44.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:44.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[106624]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11499/173103 objects degraded (6.643%); 0 B/s, 32 objects/s recovering 2024-03-29T04:49:44.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:44.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:44.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:45.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:49:44] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.43.0" 2024-03-29T04:49:45.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:45 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3984394824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:45.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:45 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3984394824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:45.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:45 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3984394824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:46.570 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:46 smithi099 ceph-mon[94036]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 11304/164238 objects degraded (6.883%); 0 B/s, 49 objects/s recovering 2024-03-29T04:49:46.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:46 smithi029 ceph-mon[102350]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 11304/164238 objects degraded (6.883%); 0 B/s, 49 objects/s recovering 2024-03-29T04:49:46.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:46 smithi029 ceph-mon[106624]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 11304/164238 objects degraded (6.883%); 0 B/s, 49 objects/s recovering 2024-03-29T04:49:47.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:47 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11304/164238 objects degraded (6.883%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:47.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:47 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11304/164238 objects degraded (6.883%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:47.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:47 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11304/164238 objects degraded (6.883%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:48.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:48 smithi099 ceph-mon[94036]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 11304/164238 objects degraded (6.883%); 0 B/s, 33 objects/s recovering 2024-03-29T04:49:48.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:48 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/612229482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:48.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:48 smithi029 ceph-mon[102350]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 11304/164238 objects degraded (6.883%); 0 B/s, 33 objects/s recovering 2024-03-29T04:49:48.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:48 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/612229482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:48.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:48 smithi029 ceph-mon[106624]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 11304/164238 objects degraded (6.883%); 0 B/s, 33 objects/s recovering 2024-03-29T04:49:48.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:48 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/612229482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:49.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:49.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:49.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:50.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:50 smithi099 ceph-mon[94036]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 11304/159732 objects degraded (7.077%); 0 B/s, 33 objects/s recovering 2024-03-29T04:49:50.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:50 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1856614891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:50.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:50 smithi029 ceph-mon[102350]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 11304/159732 objects degraded (7.077%); 0 B/s, 33 objects/s recovering 2024-03-29T04:49:50.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:50 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1856614891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:50.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:50 smithi029 ceph-mon[106624]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 11304/159732 objects degraded (7.077%); 0 B/s, 33 objects/s recovering 2024-03-29T04:49:50.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:50 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1856614891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:52.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:52 smithi099 ceph-mon[94036]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 11090/152100 objects degraded (7.291%); 0 B/s, 50 objects/s recovering 2024-03-29T04:49:52.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:52 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 11090/152100 objects degraded (7.291%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:52.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:52 smithi029 ceph-mon[102350]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 11090/152100 objects degraded (7.291%); 0 B/s, 50 objects/s recovering 2024-03-29T04:49:52.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:52 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 11090/152100 objects degraded (7.291%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:52.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:52 smithi029 ceph-mon[106624]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 11090/152100 objects degraded (7.291%); 0 B/s, 50 objects/s recovering 2024-03-29T04:49:52.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:52 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 11090/152100 objects degraded (7.291%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:53.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:53 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2831269014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:53.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:53 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2831269014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:53.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:53 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2831269014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:54.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:54 smithi099 ceph-mon[94036]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11090/149145 objects degraded (7.436%); 0 B/s, 34 objects/s recovering 2024-03-29T04:49:54.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:54 smithi029 ceph-mon[102350]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11090/149145 objects degraded (7.436%); 0 B/s, 34 objects/s recovering 2024-03-29T04:49:54.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:54 smithi029 ceph-mon[106624]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11090/149145 objects degraded (7.436%); 0 B/s, 34 objects/s recovering 2024-03-29T04:49:55.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:49:54] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2024-03-29T04:49:56.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:56 smithi099 ceph-mon[94036]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 10885/139995 objects degraded (7.775%); 0 B/s, 51 objects/s recovering 2024-03-29T04:49:56.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:56 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/314666058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:56.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:56 smithi029 ceph-mon[102350]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 10885/139995 objects degraded (7.775%); 0 B/s, 51 objects/s recovering 2024-03-29T04:49:56.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:56 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/314666058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:56.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:56 smithi029 ceph-mon[106624]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 919 op/s; 10885/139995 objects degraded (7.775%); 0 B/s, 51 objects/s recovering 2024-03-29T04:49:56.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:56 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/314666058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:57.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:57 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 10885/139995 objects degraded (7.775%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:57.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:57 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 10885/139995 objects degraded (7.775%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:57.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:57 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 10885/139995 objects degraded (7.775%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:49:58.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:58 smithi099 ceph-mon[94036]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 10885/139995 objects degraded (7.775%); 0 B/s, 34 objects/s recovering 2024-03-29T04:49:58.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:58 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1118642540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:58.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:58 smithi029 ceph-mon[102350]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 10885/139995 objects degraded (7.775%); 0 B/s, 34 objects/s recovering 2024-03-29T04:49:58.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:58 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1118642540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:58.695 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:49:58 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:49:58.428+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:58.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:58 smithi029 ceph-mon[106624]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 10885/139995 objects degraded (7.775%); 0 B/s, 34 objects/s recovering 2024-03-29T04:49:58.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:58 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1118642540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:59.625 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:59 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:59.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:59 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:59.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:49:59 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:59.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:59 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:59.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:59 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:59.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:49:59 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:49:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:59 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:59 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:49:59.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:49:59 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:00.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10885/135978 objects degraded (8.005%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:00.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 10885/135978 objects degraded (8.005%), 5 pgs degraded, 5 pgs undersized 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: [WRN] PG_DEGRADED: Degraded data redundancy: 10885/135978 objects degraded (8.005%), 5 pgs degraded, 5 pgs undersized 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: pg 8.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: pg 8.3 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [5,6] 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: pg 8.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: pg 8.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[102350]: pg 8.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10885/135978 objects degraded (8.005%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 10885/135978 objects degraded (8.005%), 5 pgs degraded, 5 pgs undersized 2024-03-29T04:50:00.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: [WRN] PG_DEGRADED: Degraded data redundancy: 10885/135978 objects degraded (8.005%), 5 pgs degraded, 5 pgs undersized 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: pg 8.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: pg 8.3 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [5,6] 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: pg 8.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: pg 8.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2024-03-29T04:50:00.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:00 smithi029 ceph-mon[106624]: pg 8.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-03-29T04:50:00.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 10885/135978 objects degraded (8.005%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 10885/135978 objects degraded (8.005%), 5 pgs degraded, 5 pgs undersized 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: daemon grafana.a on smithi099 is in error state 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: [WRN] PG_DEGRADED: Degraded data redundancy: 10885/135978 objects degraded (8.005%), 5 pgs degraded, 5 pgs undersized 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: pg 8.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,5] 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: pg 8.3 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [5,6] 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: pg 8.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: pg 8.b is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,4] 2024-03-29T04:50:00.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:00 smithi099 ceph-mon[94036]: pg 8.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2] 2024-03-29T04:50:01.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3495045553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:01.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3495045553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:01.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3495045553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:02.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:02 smithi029 ceph-mon[102350]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 10682/129171 objects degraded (8.270%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:02.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:02 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 10682/129171 objects degraded (8.270%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:02.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:02 smithi029 ceph-mon[106624]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 10682/129171 objects degraded (8.270%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:02.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:02 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 10682/129171 objects degraded (8.270%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:02.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:02 smithi099 ceph-mon[94036]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 10682/129171 objects degraded (8.270%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:02.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:02 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 10682/129171 objects degraded (8.270%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:03.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3712243395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:03.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3712243395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:03.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3712243395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:04.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:04 smithi029 ceph-mon[102350]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 10682/126213 objects degraded (8.463%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:04.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:04.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:04 smithi029 ceph-mon[106624]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 10682/126213 objects degraded (8.463%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:04.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:04.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:04 smithi099 ceph-mon[94036]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 10682/126213 objects degraded (8.463%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:04.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:05.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:50:04] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2024-03-29T04:50:06.676 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:06 smithi099 ceph-mon[94036]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 10467/117699 objects degraded (8.893%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:06.677 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3482875210' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:06.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:06 smithi029 ceph-mon[102350]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 10467/117699 objects degraded (8.893%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:06.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3482875210' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:06.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:06 smithi029 ceph-mon[106624]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 10467/117699 objects degraded (8.893%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:06.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3482875210' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:07.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:07 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 10467/117699 objects degraded (8.893%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:07.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:07 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 10467/117699 objects degraded (8.893%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:07.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:07 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 10467/117699 objects degraded (8.893%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:08.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:08 smithi029 ceph-mon[102350]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10467/117699 objects degraded (8.893%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:08.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:08 smithi029 ceph-mon[106624]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10467/117699 objects degraded (8.893%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:08.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:08 smithi099 ceph-mon[94036]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10467/117699 objects degraded (8.893%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:09.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1736940820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:09.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1736940820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:09.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1736940820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:10.695 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:10 smithi029 ceph-mon[102350]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 10467/113313 objects degraded (9.237%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:10.695 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:10 smithi029 ceph-mon[106624]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 10467/113313 objects degraded (9.237%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:10.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:10 smithi099 ceph-mon[94036]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 10467/113313 objects degraded (9.237%); 0 B/s, 34 objects/s recovering 2024-03-29T04:50:11.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:11 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/432943124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:11.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:11 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/432943124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:11.945 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:11 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/432943124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:12.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:12 smithi099 ceph-mon[94036]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 10231/105771 objects degraded (9.673%); 0 B/s, 54 objects/s recovering 2024-03-29T04:50:12.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:12 smithi099 ceph-mon[94036]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:12 smithi099 ceph-mon[94036]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:12 smithi099 ceph-mon[94036]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:12 smithi099 ceph-mon[94036]: osdmap e117: 8 total, 8 up, 8 in 2024-03-29T04:50:12.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:12 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 10231/105771 objects degraded (9.673%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:12.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[102350]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 10231/105771 objects degraded (9.673%); 0 B/s, 54 objects/s recovering 2024-03-29T04:50:12.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[102350]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[102350]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[102350]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[102350]: osdmap e117: 8 total, 8 up, 8 in 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 10231/105771 objects degraded (9.673%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[106624]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 10231/105771 objects degraded (9.673%); 0 B/s, 54 objects/s recovering 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[106624]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[106624]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[106624]: failed to encode map e117 with expected crc 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[106624]: osdmap e117: 8 total, 8 up, 8 in 2024-03-29T04:50:12.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:12 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 10231/105771 objects degraded (9.673%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:13.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:13 smithi099 ceph-mon[94036]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:13 smithi099 ceph-mon[94036]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:13 smithi099 ceph-mon[94036]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:13 smithi099 ceph-mon[94036]: osdmap e118: 8 total, 8 up, 8 in 2024-03-29T04:50:13.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:13 smithi099 ceph-mon[94036]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:13 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:13.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[102350]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[102350]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[102350]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[102350]: osdmap e118: 8 total, 8 up, 8 in 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[102350]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[106624]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[106624]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[106624]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[106624]: osdmap e118: 8 total, 8 up, 8 in 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[106624]: failed to encode map e118 with expected crc 2024-03-29T04:50:13.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:13 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:14.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:14 smithi099 ceph-mon[94036]: failed to encode map e118 with expected crc 2024-03-29T04:50:14.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:14 smithi099 ceph-mon[94036]: pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 10095/103131 objects degraded (9.789%); 0 B/s, 46 objects/s recovering 2024-03-29T04:50:14.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:14 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2405521971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:14.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-mon[102350]: failed to encode map e118 with expected crc 2024-03-29T04:50:14.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-mon[102350]: pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 10095/103131 objects degraded (9.789%); 0 B/s, 46 objects/s recovering 2024-03-29T04:50:14.825 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2405521971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:14.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-mon[106624]: failed to encode map e118 with expected crc 2024-03-29T04:50:14.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-mon[106624]: pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 10095/103131 objects degraded (9.789%); 0 B/s, 46 objects/s recovering 2024-03-29T04:50:14.825 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2405521971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:15.182 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:50:14] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2024-03-29T04:50:16.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:16 smithi029 ceph-mon[102350]: pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 10054/95949 objects degraded (10.478%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:16.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:16 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3802772463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:16.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:16 smithi029 ceph-mon[106624]: pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 10054/95949 objects degraded (10.478%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:16.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:16 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3802772463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:16.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:16 smithi099 ceph-mon[94036]: pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 905 op/s; 10054/95949 objects degraded (10.478%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:16.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:16 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3802772463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:17.737 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:17 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 10054/95949 objects degraded (10.478%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:17.737 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:17 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 10054/95949 objects degraded (10.478%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:17.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:17 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 10054/95949 objects degraded (10.478%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:18.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:18 smithi099 ceph-mon[94036]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 10054/95949 objects degraded (10.478%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:18.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:18 smithi029 ceph-mon[102350]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 10054/95949 objects degraded (10.478%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:18.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:18 smithi029 ceph-mon[106624]: pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 10054/95949 objects degraded (10.478%); 0 B/s, 51 objects/s recovering 2024-03-29T04:50:19.875 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:19.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:19.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:19 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/504708578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:19.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:19.945 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/504708578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/504708578' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:19.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:21.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:20 smithi029 ceph-mon[102350]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9908/91290 objects degraded (10.853%); 0 B/s, 40 objects/s recovering 2024-03-29T04:50:21.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:21.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:21.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:20 smithi029 ceph-mon[106624]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9908/91290 objects degraded (10.853%); 0 B/s, 40 objects/s recovering 2024-03-29T04:50:21.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:21.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:21.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:20 smithi099 ceph-mon[94036]: pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9908/91290 objects degraded (10.853%); 0 B/s, 40 objects/s recovering 2024-03-29T04:50:21.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:21.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:21.445 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:50:21.169+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:22.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 9908/85074 objects degraded (11.646%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3503094323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:22.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9908/91290 objects degraded (10.853%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 9908/85074 objects degraded (11.646%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3503094323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:22.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:21 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9908/91290 objects degraded (10.853%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 9908/85074 objects degraded (11.646%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:22.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3503094323' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:22.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:21 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9908/91290 objects degraded (10.853%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:23.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:22 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:23.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:22 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:23.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:22 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:23.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:22 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:23.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:22 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:23.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:22 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:24.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:23 smithi029 ceph-mon[102350]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9773/82278 objects degraded (11.878%); 0 B/s, 43 objects/s recovering 2024-03-29T04:50:24.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:23 smithi029 ceph-mon[106624]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9773/82278 objects degraded (11.878%); 0 B/s, 43 objects/s recovering 2024-03-29T04:50:24.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:23 smithi099 ceph-mon[94036]: pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9773/82278 objects degraded (11.878%); 0 B/s, 43 objects/s recovering 2024-03-29T04:50:25.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:24 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4039702156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:25.196 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:50:24] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.43.0" 2024-03-29T04:50:25.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:24 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4039702156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:25.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:24 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4039702156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:26.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:25 smithi099 ceph-mon[94036]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9773/77877 objects degraded (12.549%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:26.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:25 smithi029 ceph-mon[102350]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9773/77877 objects degraded (12.549%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:26.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:25 smithi029 ceph-mon[106624]: pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9773/77877 objects degraded (12.549%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:27.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:26 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9773/77877 objects degraded (12.549%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:27.223 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:26 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2492781453' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:27.224 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:26 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9773/77877 objects degraded (12.549%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:27.224 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:26 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2492781453' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:27.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:26 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9773/77877 objects degraded (12.549%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:27.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:26 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2492781453' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:28.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:27 smithi099 ceph-mon[94036]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9773/73521 objects degraded (13.293%); 0 B/s, 23 objects/s recovering 2024-03-29T04:50:28.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:27 smithi029 ceph-mon[102350]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9773/73521 objects degraded (13.293%); 0 B/s, 23 objects/s recovering 2024-03-29T04:50:28.445 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:27 smithi029 ceph-mon[106624]: pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 9773/73521 objects degraded (13.293%); 0 B/s, 23 objects/s recovering 2024-03-29T04:50:30.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:30 smithi099 ceph-mon[94036]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 9638/68847 objects degraded (13.999%); 0 B/s, 35 objects/s recovering 2024-03-29T04:50:30.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:30 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2269266400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:30.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:30 smithi029 ceph-mon[102350]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 9638/68847 objects degraded (13.999%); 0 B/s, 35 objects/s recovering 2024-03-29T04:50:30.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:30 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2269266400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:30.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:30 smithi029 ceph-mon[106624]: pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 9638/68847 objects degraded (13.999%); 0 B/s, 35 objects/s recovering 2024-03-29T04:50:30.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:30 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2269266400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:32.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:32 smithi099 ceph-mon[94036]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 9638/62679 objects degraded (15.377%); 0 B/s, 22 objects/s recovering 2024-03-29T04:50:32.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:32 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9638/62679 objects degraded (15.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:32.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:32 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1354054367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:32.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:32 smithi029 ceph-mon[102350]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 9638/62679 objects degraded (15.377%); 0 B/s, 22 objects/s recovering 2024-03-29T04:50:32.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:32 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9638/62679 objects degraded (15.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:32.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:32 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1354054367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:32.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:32 smithi029 ceph-mon[106624]: pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 9638/62679 objects degraded (15.377%); 0 B/s, 22 objects/s recovering 2024-03-29T04:50:32.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:32 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9638/62679 objects degraded (15.377%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:32.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:32 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1354054367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:34.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:34 smithi099 ceph-mon[94036]: pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 9510/59703 objects degraded (15.929%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:34.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:34.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:34 smithi029 ceph-mon[102350]: pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 9510/59703 objects degraded (15.929%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:34.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:34.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:34 smithi029 ceph-mon[106624]: pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 9510/59703 objects degraded (15.929%); 0 B/s, 33 objects/s recovering 2024-03-29T04:50:34.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:35.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:35 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1789529274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:35.196 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:50:34] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2024-03-29T04:50:35.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:35 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1789529274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:35.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:35 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1789529274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:36.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:36 smithi099 ceph-mon[94036]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9510/55431 objects degraded (17.156%); 0 B/s, 21 objects/s recovering 2024-03-29T04:50:36.393 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:36 smithi029 ceph-mon[102350]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9510/55431 objects degraded (17.156%); 0 B/s, 21 objects/s recovering 2024-03-29T04:50:36.393 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:36 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:50:36.188+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:36.394 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:36 smithi029 ceph-mon[106624]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9510/55431 objects degraded (17.156%); 0 B/s, 21 objects/s recovering 2024-03-29T04:50:37.291 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:37.291 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:37 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9510/55431 objects degraded (17.156%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:37.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:37.332 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:37 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9510/55431 objects degraded (17.156%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:37.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:37.333 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:37 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9510/55431 objects degraded (17.156%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:38.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:38 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:38.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:38 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:38.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:38 smithi099 ceph-mon[94036]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9510/51270 objects degraded (18.549%); 0 B/s, 21 objects/s recovering 2024-03-29T04:50:38.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3365526988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:38.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[102350]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9510/51270 objects degraded (18.549%); 0 B/s, 21 objects/s recovering 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3365526988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[106624]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 9510/51270 objects degraded (18.549%); 0 B/s, 21 objects/s recovering 2024-03-29T04:50:38.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3365526988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:40.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:40 smithi099 ceph-mon[94036]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 970 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 9346/46623 objects degraded (20.046%); 0 B/s, 35 objects/s recovering 2024-03-29T04:50:40.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:40 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3674628239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:40.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:40 smithi029 ceph-mon[102350]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 970 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 9346/46623 objects degraded (20.046%); 0 B/s, 35 objects/s recovering 2024-03-29T04:50:40.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:40 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3674628239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:40.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:40 smithi029 ceph-mon[106624]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 970 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 9346/46623 objects degraded (20.046%); 0 B/s, 35 objects/s recovering 2024-03-29T04:50:40.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:40 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3674628239' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:42.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:42 smithi099 ceph-mon[94036]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 829 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 9346/39855 objects degraded (23.450%); 0 B/s, 24 objects/s recovering 2024-03-29T04:50:42.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:42 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9346/39855 objects degraded (23.450%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:42.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:42 smithi029 ceph-mon[102350]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 829 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 9346/39855 objects degraded (23.450%); 0 B/s, 24 objects/s recovering 2024-03-29T04:50:42.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:42 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9346/39855 objects degraded (23.450%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:42.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:42 smithi029 ceph-mon[106624]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 829 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 9346/39855 objects degraded (23.450%); 0 B/s, 24 objects/s recovering 2024-03-29T04:50:42.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:42 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9346/39855 objects degraded (23.450%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:43.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:43 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2400613783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:43.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:43 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2400613783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:43.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:43 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2400613783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:44.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:44 smithi099 ceph-mon[94036]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9181/36834 objects degraded (24.925%); 0 B/s, 38 objects/s recovering 2024-03-29T04:50:44.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:44 smithi029 ceph-mon[102350]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9181/36834 objects degraded (24.925%); 0 B/s, 38 objects/s recovering 2024-03-29T04:50:44.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:44 smithi029 ceph-mon[106624]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 766 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9181/36834 objects degraded (24.925%); 0 B/s, 38 objects/s recovering 2024-03-29T04:50:45.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:45 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1474058409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:45.196 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:50:44] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2024-03-29T04:50:45.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:45 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1474058409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:45.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:45 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1474058409' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:46.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:46 smithi099 ceph-mon[94036]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 676 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 9181/32490 objects degraded (28.258%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:46.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:46 smithi029 ceph-mon[102350]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 676 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 9181/32490 objects degraded (28.258%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:46.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:46 smithi029 ceph-mon[106624]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 676 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 9181/32490 objects degraded (28.258%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:47.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:47 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9181/32490 objects degraded (28.258%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:47.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:47 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9181/32490 objects degraded (28.258%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:47.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:47 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9181/32490 objects degraded (28.258%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:48.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:48 smithi099 ceph-mon[94036]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 585 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 9181/28140 objects degraded (32.626%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:48.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:48 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/825173504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:48.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:48 smithi029 ceph-mon[102350]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 585 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 9181/28140 objects degraded (32.626%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:48.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:48 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/825173504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:48.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:48 smithi029 ceph-mon[106624]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 585 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 9181/28140 objects degraded (32.626%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:48.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:48 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/825173504' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:49.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:49.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:49.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:50.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:50 smithi099 ceph-mon[94036]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 472 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 9030/22680 objects degraded (39.815%); 0 B/s, 39 objects/s recovering 2024-03-29T04:50:50.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:50 smithi029 ceph-mon[102350]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 472 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 9030/22680 objects degraded (39.815%); 0 B/s, 39 objects/s recovering 2024-03-29T04:50:50.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:50 smithi029 ceph-mon[106624]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 472 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 9030/22680 objects degraded (39.815%); 0 B/s, 39 objects/s recovering 2024-03-29T04:50:51.376 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:51 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1881899491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:51.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:51 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1881899491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:51.446 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:51 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:50:51.206+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:51.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:51 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1881899491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:52.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:52 smithi029 ceph-mon[102350]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 9030/15999 objects degraded (56.441%); 0 B/s, 26 objects/s recovering 2024-03-29T04:50:52.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:52.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:52 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 9030/15999 objects degraded (56.441%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:52.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:52 smithi029 ceph-mon[106624]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 9030/15999 objects degraded (56.441%); 0 B/s, 26 objects/s recovering 2024-03-29T04:50:52.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:52.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:52 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 9030/15999 objects degraded (56.441%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:52.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:52 smithi099 ceph-mon[94036]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 332 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 9030/15999 objects degraded (56.441%); 0 B/s, 26 objects/s recovering 2024-03-29T04:50:52.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:52.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:52 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 9030/15999 objects degraded (56.441%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:53.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:53 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:53.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:53 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:53.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:53 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3493256872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:53.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:53 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:53.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:53 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:53.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:53 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3493256872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:53.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:53 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:53.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:53 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-03-29T04:50:53.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:53 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3493256872' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:54.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:54 smithi029 ceph-mon[102350]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 271 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 8847/13074 objects degraded (67.669%); 0 B/s, 41 objects/s recovering 2024-03-29T04:50:54.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:54 smithi029 ceph-mon[106624]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 271 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 8847/13074 objects degraded (67.669%); 0 B/s, 41 objects/s recovering 2024-03-29T04:50:54.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:54 smithi099 ceph-mon[94036]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 271 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 8847/13074 objects degraded (67.669%); 0 B/s, 41 objects/s recovering 2024-03-29T04:50:55.195 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:50:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:50:54] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2024-03-29T04:50:55.943 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Removed 106259 objects 2024-03-29T04:50:55.944 INFO:tasks.radosbench.radosbench.0.smithi029.stdout:Clean up completed and total clean up time :136.116 2024-03-29T04:50:56.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:56 smithi029 ceph-mon[106624]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 181 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 8847/8751 objects degraded (101.097%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:56.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:56 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2115249346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:56.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:56 smithi029 ceph-mon[102350]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 181 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 8847/8751 objects degraded (101.097%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:56.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:56 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2115249346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:56.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:56 smithi099 ceph-mon[94036]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 181 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 8847/8751 objects degraded (101.097%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:56.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:56 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2115249346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:57.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:57 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 8847/8751 objects degraded (101.097%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:57.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:57 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 8847/8751 objects degraded (101.097%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:57.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:57 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 8847/8751 objects degraded (101.097%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:50:58.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:58 smithi029 ceph-mon[102350]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 85 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 8847/4146 objects degraded (213.386%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:58.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:58 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2509452801' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:58.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:58 smithi029 ceph-mon[106624]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 85 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 8847/4146 objects degraded (213.386%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:58.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:58 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2509452801' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:58.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:58 smithi099 ceph-mon[94036]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 85 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 8847/4146 objects degraded (213.386%); 0 B/s, 27 objects/s recovering 2024-03-29T04:50:58.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:58 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2509452801' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:59.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:50:59 smithi029 ceph-mon[102350]: pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 23 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 8250/1167 objects degraded (706.941%); 0 B/s, 77 objects/s recovering 2024-03-29T04:50:59.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:50:59 smithi029 ceph-mon[106624]: pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 23 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 8250/1167 objects degraded (706.941%); 0 B/s, 77 objects/s recovering 2024-03-29T04:50:59.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:50:59 smithi099 ceph-mon[94036]: pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 23 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 8250/1167 objects degraded (706.941%); 0 B/s, 77 objects/s recovering 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: osdmap e119: 8 total, 8 up, 8 in 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[102350]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: osdmap e119: 8 total, 8 up, 8 in 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:00 smithi029 ceph-mon[106624]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.615 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-03-29T04:51:00.615 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: osdmap e119: 8 total, 8 up, 8 in 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:00.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:00 smithi099 ceph-mon[94036]: failed to encode map e119 with expected crc 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[102350]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[102350]: osdmap e120: 8 total, 8 up, 8 in 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[102350]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[102350]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2409101726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[102350]: pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 8250/72 objects degraded (11458.333%); 0 B/s, 74 objects/s recovering 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[106624]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[106624]: osdmap e120: 8 total, 8 up, 8 in 2024-03-29T04:51:01.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[106624]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[106624]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2409101726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:01.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:01 smithi029 ceph-mon[106624]: pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 8250/72 objects degraded (11458.333%); 0 B/s, 74 objects/s recovering 2024-03-29T04:51:01.618 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:01.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:01 smithi099 ceph-mon[94036]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:01 smithi099 ceph-mon[94036]: osdmap e120: 8 total, 8 up, 8 in 2024-03-29T04:51:01.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:01 smithi099 ceph-mon[94036]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:01 smithi099 ceph-mon[94036]: failed to encode map e120 with expected crc 2024-03-29T04:51:01.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2409101726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:01.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:01 smithi099 ceph-mon[94036]: pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 8250/72 objects degraded (11458.333%); 0 B/s, 74 objects/s recovering 2024-03-29T04:51:02.195 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:02 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 8250/72 objects degraded (11458.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:02.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:02 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 8250/72 objects degraded (11458.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:02.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:02 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 8250/72 objects degraded (11458.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:03.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1155857026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:03.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:03 smithi029 ceph-mon[102350]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 225 objects/s recovering 2024-03-29T04:51:03.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1155857026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:03.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:03 smithi029 ceph-mon[106624]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 225 objects/s recovering 2024-03-29T04:51:03.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1155857026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:03.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:03 smithi099 ceph-mon[94036]: pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 225 objects/s recovering 2024-03-29T04:51:04.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:04.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:04.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:05.166 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:51:04] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2024-03-29T04:51:05.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:05 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1827389158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:05.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:05 smithi029 ceph-mon[102350]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 225 objects/s recovering 2024-03-29T04:51:05.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:05 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1827389158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:05.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:05 smithi029 ceph-mon[106624]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 225 objects/s recovering 2024-03-29T04:51:05.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:05 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1827389158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:05.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:05 smithi099 ceph-mon[94036]: pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 169 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 225 objects/s recovering 2024-03-29T04:51:06.527 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:51:06.588 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:51:06.588 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:51:07.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:07 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:51:07.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:07 smithi099 ceph-mon[94036]: Upgrade: osd.2 is safe to restart 2024-03-29T04:51:07.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:07 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 7046/72 objects degraded (9786.111%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:07.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:07 smithi099 ceph-mon[94036]: pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 45 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 150 objects/s recovering 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[106624]: Upgrade: osd.2 is safe to restart 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 7046/72 objects degraded (9786.111%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[106624]: pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 45 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 150 objects/s recovering 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[102350]: Upgrade: osd.2 is safe to restart 2024-03-29T04:51:07.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 7046/72 objects degraded (9786.111%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:07.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:07 smithi029 ceph-mon[102350]: pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 45 op/s; 7046/72 objects degraded (9786.111%); 0 B/s, 150 objects/s recovering 2024-03-29T04:51:08.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:08 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3730475427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:08.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:08 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3730475427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:08.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:08 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3730475427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:09.370 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:09 smithi029 ceph-mon[102350]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 6098/72 objects degraded (8469.444%); 0 B/s, 219 objects/s recovering 2024-03-29T04:51:09.371 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:09 smithi029 ceph-mon[106624]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 6098/72 objects degraded (8469.444%); 0 B/s, 219 objects/s recovering 2024-03-29T04:51:09.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:09 smithi099 ceph-mon[94036]: pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 37 op/s; 6098/72 objects degraded (8469.444%); 0 B/s, 219 objects/s recovering 2024-03-29T04:51:10.366 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:10 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3044507164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:10.366 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:10 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3044507164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:10.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:10 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3044507164' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:11.305 INFO:teuthology.orchestra.run.smithi029.stderr:pool 'unique_pool_3' removed 2024-03-29T04:51:11.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1927260801' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:51:11.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:51:11.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: Upgrade: Updating osd.2 2024-03-29T04:51:11.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:11.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:51:11.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:11.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: Deploying daemon osd.2 on smithi029 2024-03-29T04:51:11.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:11 smithi099 ceph-mon[94036]: pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 6098/72 objects degraded (8469.444%); 0 B/s, 198 objects/s recovering 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1927260801' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: Upgrade: Updating osd.2 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: Deploying daemon osd.2 on smithi029 2024-03-29T04:51:11.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[106624]: pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 6098/72 objects degraded (8469.444%); 0 B/s, 198 objects/s recovering 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1927260801' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: Upgrade: Updating osd.2 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: Deploying daemon osd.2 on smithi029 2024-03-29T04:51:11.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:11 smithi029 ceph-mon[102350]: pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 6098/72 objects degraded (8469.444%); 0 B/s, 198 objects/s recovering 2024-03-29T04:51:12.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:12 smithi099 ceph-mon[94036]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:51:12.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:12 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:12.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:12 smithi099 ceph-mon[94036]: osdmap e121: 8 total, 8 up, 8 in 2024-03-29T04:51:12.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:12 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 6098/72 objects degraded (8469.444%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:12.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:12 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:12.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:12 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2071781629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[102350]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[102350]: osdmap e121: 8 total, 8 up, 8 in 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 6098/72 objects degraded (8469.444%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2071781629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:12.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[106624]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-03-29T04:51:12.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:12.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[106624]: osdmap e121: 8 total, 8 up, 8 in 2024-03-29T04:51:12.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 6098/72 objects degraded (8469.444%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-03-29T04:51:12.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:12.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:12 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2071781629' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:13.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:13 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:13 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:13 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:13 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:13 smithi099 ceph-mon[94036]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:13 smithi099 ceph-mon[94036]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:13.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[102350]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[102350]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:13.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[106624]: failed to encode map e121 with expected crc 2024-03-29T04:51:13.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:13 smithi029 ceph-mon[106624]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:14.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:14 smithi099 ceph-mon[94036]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6098/72 objects degraded (8469.444%), 3 pgs degraded, 3 pgs undersized) 2024-03-29T04:51:14.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:14 smithi029 ceph-mon[102350]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6098/72 objects degraded (8469.444%), 3 pgs degraded, 3 pgs undersized) 2024-03-29T04:51:14.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:14 smithi029 ceph-mon[106624]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6098/72 objects degraded (8469.444%), 3 pgs degraded, 3 pgs undersized) 2024-03-29T04:51:15.077 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:51:14] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2024-03-29T04:51:15.103 INFO:teuthology.task.sequential:In sequential, running task print... 2024-03-29T04:51:15.103 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-03-29T04:51:15.103 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:15.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/71080722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:15.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:15 smithi029 ceph-mon[102350]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:15.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/71080722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:15.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:15 smithi029 ceph-mon[106624]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:15.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/71080722' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:15.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:15 smithi099 ceph-mon[94036]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:17.061 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:16 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/776599367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:17.062 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:16 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/776599367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:17.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:16 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/776599367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:17.830 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:17 smithi029 ceph-mon[102350]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:17.831 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:17 smithi029 ceph-mon[106624]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:17.831 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:17 smithi029 systemd[1]: Stopping Ceph osd.2 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:51:18.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:17 smithi099 ceph-mon[94036]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:18.196 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[58768]: 2024-03-29T04:51:17.825+0000 7f05238d9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:51:18.196 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[58768]: 2024-03-29T04:51:17.825+0000 7f05238d9700 -1 osd.2 121 *** Got signal Terminated *** 2024-03-29T04:51:18.196 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:17 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[58768]: 2024-03-29T04:51:17.825+0000 7f05238d9700 -1 osd.2 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:51:19.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:18 smithi099 ceph-mon[94036]: osd.2 marked itself down and dead 2024-03-29T04:51:19.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:18 smithi029 ceph-mon[102350]: osd.2 marked itself down and dead 2024-03-29T04:51:19.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:18 smithi029 ceph-mon[106624]: osd.2 marked itself down and dead 2024-03-29T04:51:19.540 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:19 smithi029 podman[133194]: 2024-03-29 04:51:19.212146222 +0000 UTC m=+1.520573124 container died 817a836d7edf8440eea6ee22edf320241fb2ee2f3d6e3d3cf8bdf847ac89088a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-29T04:51:19.918 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:19 smithi029 podman[133194]: 2024-03-29 04:51:19.644537007 +0000 UTC m=+1.952963901 container cleanup 817a836d7edf8440eea6ee22edf320241fb2ee2f3d6e3d3cf8bdf847ac89088a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17) 2024-03-29T04:51:19.918 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:19 smithi029 bash[133194]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2 2024-03-29T04:51:19.918 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: osdmap e122: 8 total, 7 up, 8 in 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: pgmap v438: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3465106192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:19.919 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: osdmap e122: 8 total, 7 up, 8 in 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: failed to encode map e122 with expected crc 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: pgmap v438: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3465106192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:19.920 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:20.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: osdmap e122: 8 total, 7 up, 8 in 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: failed to encode map e122 with expected crc 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: pgmap v438: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3465106192' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:20.196 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:19 smithi029 podman[133243]: 2024-03-29 04:51:19.914753151 +0000 UTC m=+0.695146993 container remove 817a836d7edf8440eea6ee22edf320241fb2ee2f3d6e3d3cf8bdf847ac89088a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:51:20.628 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:20 smithi029 podman[133339]: 2024-03-29 04:51:20.341965431 +0000 UTC m=+0.128710398 container create acbe21bdb2ed177aa58b0a059a20d526dc2e4361785918b026dfba817376bcc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326) 2024-03-29T04:51:20.628 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:20 smithi029 podman[133339]: 2024-03-29 04:51:20.245964523 +0000 UTC m=+0.032709472 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:20.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:51:20.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: osdmap e123: 8 total, 7 up, 8 in 2024-03-29T04:51:20.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[102350]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: osdmap e123: 8 total, 7 up, 8 in 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:20 smithi029 ceph-mon[106624]: failed to encode map e123 with expected crc 2024-03-29T04:51:20.948 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:20 smithi029 podman[133339]: 2024-03-29 04:51:20.624341522 +0000 UTC m=+0.411086443 container init acbe21bdb2ed177aa58b0a059a20d526dc2e4361785918b026dfba817376bcc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362) 2024-03-29T04:51:20.948 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:20 smithi029 podman[133339]: 2024-03-29 04:51:20.629700185 +0000 UTC m=+0.416445107 container start acbe21bdb2ed177aa58b0a059a20d526dc2e4361785918b026dfba817376bcc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-29T04:51:20.948 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:20 smithi029 podman[133339]: 2024-03-29 04:51:20.740636143 +0000 UTC m=+0.527381074 container attach acbe21bdb2ed177aa58b0a059a20d526dc2e4361785918b026dfba817376bcc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:51:21.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:51:21.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: osdmap e123: 8 total, 7 up, 8 in 2024-03-29T04:51:21.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:20 smithi099 ceph-mon[94036]: failed to encode map e123 with expected crc 2024-03-29T04:51:21.273 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:20 smithi029 podman[133339]: 2024-03-29 04:51:20.971762176 +0000 UTC m=+0.758507109 container died acbe21bdb2ed177aa58b0a059a20d526dc2e4361785918b026dfba817376bcc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-8da2362, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-03-29T04:51:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:21 smithi029 ceph-mon[102350]: pgmap v440: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 5 peering, 2 stale+active+clean, 7 active+undersized, 57 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:21.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2753167966' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:21.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:21 smithi029 ceph-mon[106624]: pgmap v440: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 5 peering, 2 stale+active+clean, 7 active+undersized, 57 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:21.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:21 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2753167966' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:21.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:21 smithi029 podman[133385]: 2024-03-29 04:51:21.566310749 +0000 UTC m=+0.589311484 container remove acbe21bdb2ed177aa58b0a059a20d526dc2e4361785918b026dfba817376bcc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-03-29T04:51:21.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:21 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-29T04:51:21.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:21 smithi029 systemd[1]: Stopped Ceph osd.2 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:51:21.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:21 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.2.service: Consumed 25min 15.994s CPU time. 2024-03-29T04:51:21.947 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:21 smithi029 systemd[1]: Starting Ceph osd.2 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:51:22.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:21 smithi099 ceph-mon[94036]: pgmap v440: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 5 peering, 2 stale+active+clean, 7 active+undersized, 57 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:22.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:21 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2753167966' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:22.353 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 podman[133524]: 2024-03-29 04:51:21.932417414 +0000 UTC m=+0.018072019 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:22.354 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 podman[133524]: 2024-03-29 04:51:22.0689505 +0000 UTC m=+0.154605107 container create 73b965f0048245f3a6037fb12b6ed0ad8f8f8a6c3891d647c1f66a7adf25c8c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-03-29T04:51:22.696 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 podman[133524]: 2024-03-29 04:51:22.349973951 +0000 UTC m=+0.435628556 container init 73b965f0048245f3a6037fb12b6ed0ad8f8f8a6c3891d647c1f66a7adf25c8c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-03-29T04:51:22.696 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 podman[133524]: 2024-03-29 04:51:22.355973117 +0000 UTC m=+0.441627724 container start 73b965f0048245f3a6037fb12b6ed0ad8f8f8a6c3891d647c1f66a7adf25c8c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:51:22.696 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 podman[133524]: 2024-03-29 04:51:22.475231342 +0000 UTC m=+0.560885950 container attach 73b965f0048245f3a6037fb12b6ed0ad8f8f8a6c3891d647c1f66a7adf25c8c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-29T04:51:23.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:22 smithi099 ceph-mon[94036]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 1 pg degraded (PG_DEGRADED) 2024-03-29T04:51:23.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-mon[106624]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 1 pg degraded (PG_DEGRADED) 2024-03-29T04:51:23.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-mon[102350]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 1 pg degraded (PG_DEGRADED) 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 bash[133524]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:51:23.197 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate[133539]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-29T04:51:23.198 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:22 smithi029 podman[133524]: 2024-03-29 04:51:22.881220381 +0000 UTC m=+0.966874985 container died 73b965f0048245f3a6037fb12b6ed0ad8f8f8a6c3891d647c1f66a7adf25c8c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:51:23.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:23 smithi029 ceph-mon[102350]: pgmap v441: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 5 peering, 12 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:23.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:23 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1011967718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:23.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:23 smithi029 ceph-mon[106624]: pgmap v441: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 5 peering, 12 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:23.940 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:23 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1011967718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:23.940 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:23 smithi029 podman[133685]: 2024-03-29 04:51:23.584838398 +0000 UTC m=+0.694210079 container remove 73b965f0048245f3a6037fb12b6ed0ad8f8f8a6c3891d647c1f66a7adf25c8c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:51:23.940 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:23 smithi029 podman[133781]: 2024-03-29 04:51:23.697860588 +0000 UTC m=+0.017212700 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:23.940 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:23 smithi029 podman[133781]: 2024-03-29 04:51:23.820664901 +0000 UTC m=+0.140016993 container create 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-29T04:51:24.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:23 smithi099 ceph-mon[94036]: pgmap v441: 73 pgs: 1 activating+undersized, 1 active+undersized+degraded, 5 peering, 12 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:24.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:23 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1011967718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:24.446 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:24 smithi029 podman[133781]: 2024-03-29 04:51:24.042962027 +0000 UTC m=+0.362314136 container init 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362) 2024-03-29T04:51:24.446 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:24 smithi029 podman[133781]: 2024-03-29 04:51:24.048402982 +0000 UTC m=+0.367755074 container start 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-29T04:51:24.446 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:24 smithi029 ceph-osd[133818]: -- 172.21.15.29:0/3416426129 <== mon.0 v2:172.21.15.29:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556a7e8d3680 con 0x556a7e89d000 2024-03-29T04:51:24.447 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:24 smithi029 bash[133781]: 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd 2024-03-29T04:51:24.447 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:24 smithi029 systemd[1]: Started Ceph osd.2 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:51:24.946 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:51:24] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.43.0" 2024-03-29T04:51:24.946 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[133795]: 2024-03-29T04:51:24.657+0000 7f2be753a7c0 -1 Falling back to public interface 2024-03-29T04:51:25.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[102350]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:25.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:25 smithi029 ceph-mon[106624]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:25.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:25 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:25.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:25 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:25.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:25 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:25.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:25 smithi099 ceph-mon[94036]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:26.490 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:26 smithi029 ceph-mon[102350]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-29T04:51:26.490 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:26 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3886197581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:26.490 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:26 smithi029 ceph-mon[106624]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-29T04:51:26.490 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:26 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3886197581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:26.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:26 smithi099 ceph-mon[94036]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-29T04:51:26.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:26 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3886197581' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:27.626 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:27 smithi099 ceph-mon[94036]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:27.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:27 smithi029 ceph-mon[102350]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:27.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:27 smithi029 ceph-mon[106624]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:29.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:29 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:29 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:29 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/469013037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:29.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:29 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:29 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:29 smithi099 ceph-mon[94036]: pgmap v444: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:29.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.698 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/469013037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[102350]: pgmap v444: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/469013037' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:29.699 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:29.700 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:29 smithi029 ceph-mon[106624]: pgmap v444: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:30.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:30 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:30.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:30 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:30.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:30 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:30.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:30 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:30.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:30 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:30.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:30 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:31.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:31 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/715786105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:31.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:31 smithi029 ceph-mon[102350]: pgmap v445: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:31.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:31 smithi029 ceph-mon[102350]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:31.696 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:31 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/715786105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:31.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:31 smithi029 ceph-mon[106624]: pgmap v445: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:31.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:31 smithi029 ceph-mon[106624]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:31.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:31 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/715786105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:31.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:31 smithi099 ceph-mon[94036]: pgmap v445: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:31.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:31 smithi099 ceph-mon[94036]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:32.446 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[133795]: 2024-03-29T04:51:32.235+0000 7f2be753a7c0 -1 osd.2 121 log_to_monitors true 2024-03-29T04:51:32.937 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:32 smithi029 ceph-mon[102350]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:51:32.937 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:32 smithi029 ceph-mon[106624]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:51:33.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:32 smithi099 ceph-mon[94036]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:51:33.196 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:51:32 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[133795]: 2024-03-29T04:51:32.932+0000 7f2bd498a700 -1 osd.2 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:51:33.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:51:33.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: osdmap e124: 8 total, 7 up, 8 in 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: pgmap v447: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/472024184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:33.947 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:51:33.850+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: osdmap e124: 8 total, 7 up, 8 in 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: failed to encode map e124 with expected crc 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: pgmap v447: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:33.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/472024184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:34.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: osdmap e124: 8 total, 7 up, 8 in 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: from='osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196]' entity='osd.2' 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: pgmap v447: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:34.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/472024184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:34.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:34.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: pgmap v448: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: pgmap v449: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196] boot 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: osdmap e125: 8 total, 8 up, 8 in 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: failed to encode map e124 with expected crc 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: pgmap v448: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: pgmap v449: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196] boot 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: osdmap e125: 8 total, 8 up, 8 in 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: failed to encode map e125 with expected crc 2024-03-29T04:51:34.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:34.913 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: failed to encode map e124 with expected crc 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: pgmap v448: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: pgmap v449: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:35.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: failed to encode map e125 with expected crc 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: failed to encode map e125 with expected crc 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: osd.2 [v2:172.21.15.29:6818/3633025196,v1:172.21.15.29:6819/3633025196] boot 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: osdmap e125: 8 total, 8 up, 8 in 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:51:35.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: failed to encode map e125 with expected crc 2024-03-29T04:51:35.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: failed to encode map e125 with expected crc 2024-03-29T04:51:35.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:35.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:35.196 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:51:34] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.43.0" 2024-03-29T04:51:35.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: failed to encode map e125 with expected crc 2024-03-29T04:51:35.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: osdmap e126: 8 total, 8 up, 8 in 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2904206686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: failed to encode map e125 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: osdmap e126: 8 total, 8 up, 8 in 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:35.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:35 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2904206686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: failed to encode map e125 with expected crc 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: osdmap e126: 8 total, 8 up, 8 in 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.028 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:35 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2904206686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:36.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[102350]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[102350]: pgmap v452: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:36.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[106624]: failed to encode map e126 with expected crc 2024-03-29T04:51:36.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:36 smithi029 ceph-mon[106624]: pgmap v452: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:37.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:36 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:37.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:36 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:37.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:36 smithi099 ceph-mon[94036]: failed to encode map e126 with expected crc 2024-03-29T04:51:37.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:36 smithi099 ceph-mon[94036]: pgmap v452: 73 pgs: 2 peering, 10 active+undersized, 61 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:37.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:37 smithi029 ceph-mon[102350]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2024-03-29T04:51:38.037 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:37 smithi029 ceph-mon[106624]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2024-03-29T04:51:38.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:37 smithi099 ceph-mon[94036]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded) 2024-03-29T04:51:38.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:38 smithi029 ceph-mon[102350]: pgmap v453: 73 pgs: 2 peering, 4 active+undersized, 67 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:38.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3204084939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:38.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:38 smithi029 ceph-mon[106624]: pgmap v453: 73 pgs: 2 peering, 4 active+undersized, 67 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:38.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3204084939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:39.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:38 smithi099 ceph-mon[94036]: pgmap v453: 73 pgs: 2 peering, 4 active+undersized, 67 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:39.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3204084939' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:41.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:40 smithi099 ceph-mon[94036]: pgmap v454: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:41.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:40 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3729249659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:41.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:40 smithi029 ceph-mon[102350]: pgmap v454: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:41.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:40 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3729249659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:41.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:40 smithi029 ceph-mon[106624]: pgmap v454: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:41.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:40 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3729249659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:43.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:42 smithi099 ceph-mon[94036]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:43.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:42 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1130700968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:43.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:42 smithi029 ceph-mon[102350]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:43.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:42 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1130700968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:43.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:42 smithi029 ceph-mon[106624]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:43.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:42 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1130700968' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:45.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:44 smithi099 ceph-mon[94036]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:45.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:44 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3531436908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:45.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:44 smithi029 ceph-mon[106624]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:45.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:44 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3531436908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:45.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:44 smithi029 ceph-mon[102350]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:45.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:44 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3531436908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:45.197 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:51:44] "GET /metrics HTTP/1.1" 200 33939 "" "Prometheus/2.43.0" 2024-03-29T04:51:47.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:46 smithi099 ceph-mon[94036]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:47.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:46 smithi029 ceph-mon[106624]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:47.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:46 smithi029 ceph-mon[102350]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:48.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:47 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1481065740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:48.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:47 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1481065740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:48.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:47 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1481065740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:49.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:48 smithi099 ceph-mon[94036]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:49.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:48 smithi029 ceph-mon[102350]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:49.196 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:48 smithi029 ceph-mon[106624]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:50.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:50.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:50.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: Upgrade: osd.3 is safe to restart 2024-03-29T04:51:50.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:50.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:49 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/973819609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:50.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:50.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: Upgrade: osd.3 is safe to restart 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/973819609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: Upgrade: osd.3 is safe to restart 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:49 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/973819609' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:50.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[106624]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:50.946 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:51:50.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:50.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[102350]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:50.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:50.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:51:50.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:51.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:50 smithi099 ceph-mon[94036]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:51.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:51:51.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:51:51.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:51.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:51 smithi029 ceph-mon[102350]: Upgrade: Updating osd.3 2024-03-29T04:51:51.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:51 smithi029 ceph-mon[102350]: Deploying daemon osd.3 on smithi029 2024-03-29T04:51:51.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:51 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2058088130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:51.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:51 smithi029 ceph-mon[106624]: Upgrade: Updating osd.3 2024-03-29T04:51:51.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:51 smithi029 ceph-mon[106624]: Deploying daemon osd.3 on smithi029 2024-03-29T04:51:51.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:51 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2058088130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:52.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:51 smithi099 ceph-mon[94036]: Upgrade: Updating osd.3 2024-03-29T04:51:52.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:51 smithi099 ceph-mon[94036]: Deploying daemon osd.3 on smithi029 2024-03-29T04:51:52.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:51 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2058088130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:53.086 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:52 smithi029 ceph-mon[102350]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:53.087 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:52 smithi029 ceph-mon[106624]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:53.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:52 smithi099 ceph-mon[94036]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:54.667 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:54 smithi029 systemd[1]: Stopping Ceph osd.3 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:51:54.946 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:51:54] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:51:54.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-mon[102350]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:54.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2790154436' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:54.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-mon[102350]: osd.3 marked itself down and dead 2024-03-29T04:51:54.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-mon[106624]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:54.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2790154436' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:54.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-mon[106624]: osd.3 marked itself down and dead 2024-03-29T04:51:54.947 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[65409]: 2024-03-29T04:51:54.662+0000 7efd7d33a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:51:54.948 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[65409]: 2024-03-29T04:51:54.662+0000 7efd7d33a700 -1 osd.3 126 *** Got signal Terminated *** 2024-03-29T04:51:54.948 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[65409]: 2024-03-29T04:51:54.662+0000 7efd7d33a700 -1 osd.3 126 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:51:55.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:54 smithi099 ceph-mon[94036]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:55.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:54 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2790154436' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:55.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:54 smithi099 ceph-mon[94036]: osd.3 marked itself down and dead 2024-03-29T04:51:55.696 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:55 smithi029 podman[138785]: 2024-03-29 04:51:55.418601121 +0000 UTC m=+0.872695767 container died a4cb24ddb4069ca4d68b15ba7cce71059f319e09a25b2ca2c708c9bc670915b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:51:56.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: osdmap e127: 8 total, 7 up, 8 in 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:55 smithi099 ceph-mon[94036]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.196 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:55 smithi029 podman[138785]: 2024-03-29 04:51:55.906543165 +0000 UTC m=+1.360637810 container cleanup a4cb24ddb4069ca4d68b15ba7cce71059f319e09a25b2ca2c708c9bc670915b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:51:56.196 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:55 smithi029 bash[138785]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: osdmap e127: 8 total, 7 up, 8 in 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[102350]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:51:56.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: osdmap e127: 8 total, 7 up, 8 in 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:55 smithi029 ceph-mon[106624]: failed to encode map e127 with expected crc 2024-03-29T04:51:56.531 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:56 smithi029 podman[138807]: 2024-03-29 04:51:56.201384955 +0000 UTC m=+0.778734041 container remove a4cb24ddb4069ca4d68b15ba7cce71059f319e09a25b2ca2c708c9bc670915b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:51:56.792 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:56 smithi029 podman[138924]: 2024-03-29 04:51:56.430915764 +0000 UTC m=+0.025379389 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:56.792 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:56 smithi029 podman[138924]: 2024-03-29 04:51:56.612266996 +0000 UTC m=+0.206730607 container create 0e81098451e7a80cd5526b4e516826f2587b90d6d9cf70edd98a341d73798139 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: pgmap v463: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: osdmap e128: 8 total, 7 up, 8 in 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:56 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1811066839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:57.144 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:56 smithi029 podman[138924]: 2024-03-29 04:51:56.835693489 +0000 UTC m=+0.430157110 container init 0e81098451e7a80cd5526b4e516826f2587b90d6d9cf70edd98a341d73798139 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-8da2362, org.label-schema.license=GPLv2) 2024-03-29T04:51:57.144 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:56 smithi029 podman[138924]: 2024-03-29 04:51:56.841804351 +0000 UTC m=+0.436267968 container start 0e81098451e7a80cd5526b4e516826f2587b90d6d9cf70edd98a341d73798139 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:51:57.144 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:56 smithi029 podman[138924]: 2024-03-29 04:51:56.960586875 +0000 UTC m=+0.555050485 container attach 0e81098451e7a80cd5526b4e516826f2587b90d6d9cf70edd98a341d73798139 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-03-29T04:51:57.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: pgmap v463: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:57.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-03-29T04:51:57.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.144 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: osdmap e128: 8 total, 7 up, 8 in 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1811066839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: pgmap v463: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.145 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: osdmap e128: 8 total, 7 up, 8 in 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: failed to encode map e128 with expected crc 2024-03-29T04:51:57.146 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:56 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1811066839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:57.420 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:57 smithi029 podman[138924]: 2024-03-29 04:51:57.140293976 +0000 UTC m=+0.734757586 container died 0e81098451e7a80cd5526b4e516826f2587b90d6d9cf70edd98a341d73798139 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True) 2024-03-29T04:51:58.196 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:57 smithi029 podman[138973]: 2024-03-29 04:51:57.828405397 +0000 UTC m=+0.683170683 container remove 0e81098451e7a80cd5526b4e516826f2587b90d6d9cf70edd98a341d73798139 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:51:58.197 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:57 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-29T04:51:58.197 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:57 smithi029 systemd[1]: Stopped Ceph osd.3 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:51:58.197 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:57 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.3.service: Consumed 27min 49.681s CPU time. 2024-03-29T04:51:58.197 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:58 smithi029 systemd[1]: Starting Ceph osd.3 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:51:58.574 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:58 smithi029 podman[139090]: 2024-03-29 04:51:58.172988707 +0000 UTC m=+0.018674260 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:58.574 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:58 smithi029 podman[139090]: 2024-03-29 04:51:58.314132623 +0000 UTC m=+0.159818164 container create e93a4065eab61c957c027834ae20ee4dc01285206f8457df2b89ce590f000e8c (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-29T04:51:58.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:58 smithi029 ceph-mon[102350]: pgmap v465: 73 pgs: 14 peering, 5 stale+active+clean, 5 active+undersized, 49 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:58.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:51:58 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/823875491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:58.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:58 smithi029 ceph-mon[106624]: pgmap v465: 73 pgs: 14 peering, 5 stale+active+clean, 5 active+undersized, 49 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:58.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:51:58 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/823875491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:58.947 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:58 smithi029 podman[139090]: 2024-03-29 04:51:58.570720635 +0000 UTC m=+0.416406201 container init e93a4065eab61c957c027834ae20ee4dc01285206f8457df2b89ce590f000e8c (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True) 2024-03-29T04:51:58.947 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:58 smithi029 podman[139090]: 2024-03-29 04:51:58.576643904 +0000 UTC m=+0.422329448 container start e93a4065eab61c957c027834ae20ee4dc01285206f8457df2b89ce590f000e8c (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate, GIT_CLEAN=True, ceph=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T04:51:58.947 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:58 smithi029 podman[139090]: 2024-03-29 04:51:58.687493443 +0000 UTC m=+0.533178991 container attach e93a4065eab61c957c027834ae20ee4dc01285206f8457df2b89ce590f000e8c (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:51:59.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:58 smithi099 ceph-mon[94036]: pgmap v465: 73 pgs: 14 peering, 5 stale+active+clean, 5 active+undersized, 49 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:51:59.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:51:58 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/823875491' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:59.430 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 bash[139090]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:59.431 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate[139106]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-29T04:51:59.432 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 podman[139090]: 2024-03-29 04:51:59.056311302 +0000 UTC m=+0.901996843 container died e93a4065eab61c957c027834ae20ee4dc01285206f8457df2b89ce590f000e8c (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:52:00.131 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 podman[139269]: 2024-03-29 04:51:59.73043506 +0000 UTC m=+0.668886106 container remove e93a4065eab61c957c027834ae20ee4dc01285206f8457df2b89ce590f000e8c (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-29T04:52:00.131 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 podman[139336]: 2024-03-29 04:51:59.852339512 +0000 UTC m=+0.017826267 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:52:00.131 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:51:59 smithi029 podman[139336]: 2024-03-29 04:51:59.974205512 +0000 UTC m=+0.139692255 container create e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, ceph=True) 2024-03-29T04:52:00.696 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:00 smithi029 podman[139336]: 2024-03-29 04:52:00.247811394 +0000 UTC m=+0.413298152 container init e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:52:00.697 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:00 smithi029 podman[139336]: 2024-03-29 04:52:00.252898942 +0000 UTC m=+0.418385688 container start e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.build-date=20240326) 2024-03-29T04:52:00.697 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-osd[139374]: -- 172.21.15.29:0/959390650 <== mon.2 v2:172.21.15.29:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5565c550d680 con 0x5565c54d6c00 2024-03-29T04:52:00.697 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:00 smithi029 bash[139336]: e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 2024-03-29T04:52:00.697 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:00 smithi029 systemd[1]: Started Ceph osd.3 for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:52:01.126 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:00 smithi099 ceph-mon[94036]: pgmap v466: 73 pgs: 14 peering, 2 stale+active+clean, 10 active+undersized, 47 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:52:01.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:00 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:01.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:00 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:01.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:00 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:01.197 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:52:00.791+0000 7f572eb787c0 -1 Falling back to public interface 2024-03-29T04:52:01.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[102350]: pgmap v466: 73 pgs: 14 peering, 2 stale+active+clean, 10 active+undersized, 47 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:52:01.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:01.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:01.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:01.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[106624]: pgmap v466: 73 pgs: 14 peering, 2 stale+active+clean, 10 active+undersized, 47 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:52:01.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:01.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:01.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:00 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:02.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3446880419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:02.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3446880419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:02.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3446880419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:03.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:02 smithi099 ceph-mon[94036]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:03.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:02 smithi099 ceph-mon[94036]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:52:03.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:02 smithi099 ceph-mon[94036]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-03-29T04:52:03.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:02 smithi029 ceph-mon[102350]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:03.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:02 smithi029 ceph-mon[102350]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:52:03.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:02 smithi029 ceph-mon[102350]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-03-29T04:52:03.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:02 smithi029 ceph-mon[106624]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:03.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:02 smithi029 ceph-mon[106624]: Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:52:03.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:02 smithi029 ceph-mon[106624]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-03-29T04:52:04.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1031428421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:04.185 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1031428421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:04.186 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1031428421' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:05.196 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:52:04] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[102350]: pgmap v468: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[106624]: pgmap v468: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:04 smithi099 ceph-mon[94036]: pgmap v468: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:05.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:05.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:05.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:06.603 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:52:06.246+0000 7f572eb787c0 -1 osd.3 126 log_to_monitors true 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[102350]: pgmap v469: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2992442658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[102350]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[106624]: pgmap v469: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:06.859 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2992442658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:06.860 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:06 smithi029 ceph-mon[106624]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:52:07.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:06 smithi099 ceph-mon[94036]: pgmap v469: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:07.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:07.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:06 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:07.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2992442658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:07.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:06 smithi099 ceph-mon[94036]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:52:07.196 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:52:07.010+0000 7f571ffd0700 -1 osd.3 126 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:52:08.186 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 4 2024-03-29T04:52:08.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-mon[102350]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:52:08.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-mon[102350]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-mon[102350]: osdmap e129: 8 total, 7 up, 8 in 2024-03-29T04:52:08.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-mon[102350]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:52:08.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-mon[102350]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.263 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:07 smithi029 ceph-mon[102350]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:08 smithi029 ceph-mon[106624]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:52:08.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:08 smithi029 ceph-mon[106624]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:08 smithi029 ceph-mon[106624]: osdmap e129: 8 total, 7 up, 8 in 2024-03-29T04:52:08.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:08 smithi029 ceph-mon[106624]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:52:08.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:08 smithi029 ceph-mon[106624]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.264 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:08 smithi029 ceph-mon[106624]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:07 smithi099 ceph-mon[94036]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:52:08.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:07 smithi099 ceph-mon[94036]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:07 smithi099 ceph-mon[94036]: osdmap e129: 8 total, 7 up, 8 in 2024-03-29T04:52:08.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:07 smithi099 ceph-mon[94036]: from='osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi029", "root=default"]}]: dispatch 2024-03-29T04:52:08.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:07 smithi099 ceph-mon[94036]: failed to encode map e129 with expected crc 2024-03-29T04:52:08.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:07 smithi099 ceph-mon[94036]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: pgmap v471: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653] boot 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: osdmap e130: 8 total, 8 up, 8 in 2024-03-29T04:52:09.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:52:09.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3071981410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:09.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: pgmap v471: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653] boot 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: osdmap e130: 8 total, 8 up, 8 in 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3071981410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e129 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: pgmap v471: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: osd.3 [v2:172.21.15.29:6826/3635564653,v1:172.21.15.29:6827/3635564653] boot 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: osdmap e130: 8 total, 8 up, 8 in 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: failed to encode map e130 with expected crc 2024-03-29T04:52:09.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3071981410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:52:09.470 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:52:10.083 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:10.215 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:52:10.179+0000 7f94b7a0c700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: osdmap e131: 8 total, 8 up, 8 in 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[102350]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.216 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: osdmap e131: 8 total, 8 up, 8 in 2024-03-29T04:52:10.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.217 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:10 smithi029 ceph-mon[106624]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: osdmap e131: 8 total, 8 up, 8 in 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:10.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:10 smithi099 ceph-mon[94036]: failed to encode map e131 with expected crc 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: pgmap v474: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: pgmap v475: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: pgmap v476: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:11.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:11 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: pgmap v474: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: pgmap v475: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: pgmap v476: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: pgmap v474: 73 pgs: 2 peering, 3 active+undersized+degraded, 26 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:11.380 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: pgmap v475: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: pgmap v476: 73 pgs: 2 peering, 3 active+undersized+degraded, 21 active+undersized, 47 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:11.381 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:11 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:52:12.182 INFO:teuthology.orchestra.run.smithi029.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:52:12.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:12 smithi029 ceph-mon[102350]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:52:12.194 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:12 smithi029 ceph-mon[102350]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-03-29T04:52:12.194 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:12 smithi029 ceph-mon[106624]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:52:12.195 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:12 smithi029 ceph-mon[106624]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-03-29T04:52:12.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:12 smithi099 ceph-mon[94036]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:52:12.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:12 smithi099 ceph-mon[94036]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-03-29T04:52:12.857 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:52:13.125 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:13.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:13 smithi029 ceph-mon[102350]: pgmap v477: 73 pgs: 2 peering, 3 active+undersized, 68 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:13.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:13 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:13.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:13 smithi029 ceph-mon[106624]: pgmap v477: 73 pgs: 2 peering, 3 active+undersized, 68 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:13.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:13 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:13.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:13 smithi099 ceph-mon[94036]: pgmap v477: 73 pgs: 2 peering, 3 active+undersized, 68 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:13.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:13 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:14.319 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-mon[102350]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:14.319 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-mon[102350]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:52:14.319 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-mon[102350]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-03-29T04:52:14.319 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-mon[106624]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:14.319 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-mon[106624]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:52:14.319 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-mon[106624]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-03-29T04:52:14.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:14 smithi099 ceph-mon[94036]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:14.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:14 smithi099 ceph-mon[94036]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:52:14.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:14 smithi099 ceph-mon[94036]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-03-29T04:52:15.026 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:52:14] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2024-03-29T04:52:15.317 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:15 smithi029 ceph-mon[102350]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:15.317 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:15 smithi029 ceph-mon[106624]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:15.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:15 smithi099 ceph-mon[94036]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:15.534 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:52:15.534 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (15m) 10s ago 20m 22.9M - 0.25.0 c8568f914cd2 6fb7bc148154 2024-03-29T04:52:15.534 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi099 *:3000 error 47s ago 20m - - 2024-03-29T04:52:15.534 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.gsovzu smithi029 running (19m) 10s ago 19m 19.4M - 17.2.7-789-g4213e17c cf22eb8683be e6b24be4eed9 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi099.jhvkhg smithi099 running (19m) 47s ago 19m 18.8M - 17.2.7-789-g4213e17c cf22eb8683be 2d2ff55bb03a 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi099 *:8443,9283,8765 running (14m) 47s ago 24m 455M - 19.0.0-1959-g8da23624 54112f57da44 1c6cc4c8d8c3 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (15m) 10s ago 26m 510M - 19.0.0-1959-g8da23624 54112f57da44 280a22bdfead 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (13m) 10s ago 27m 67.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 4ef356385530 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi099 running (10m) 47s ago 25m 46.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 a53b591d5ed7 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (11m) 10s ago 25m 51.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 69712aa77fae 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (15m) 10s ago 20m 16.6M - 1.5.0 0da6a335fe13 9026c9798e07 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi099 *:9100 running (15m) 47s ago 20m 15.5M - 1.5.0 0da6a335fe13 52f151f28461 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (8m) 10s ago 24m 685M 2481M 19.0.0-1959-g8da23624 54112f57da44 8456d3dfc607 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (5m) 10s ago 24m 249M 2481M 19.0.0-1959-g8da23624 54112f57da44 cd71767b2146 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (51s) 10s ago 23m 53.4M 2481M 19.0.0-1959-g8da23624 54112f57da44 7299367ca14b 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (15s) 10s ago 23m 13.8M 2481M 19.0.0-1959-g8da23624 54112f57da44 e0e58b4f3af9 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi099 running (22m) 47s ago 22m 525M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1fab66997673 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi099 running (22m) 47s ago 22m 983M 2480M 17.2.7-789-g4213e17c cf22eb8683be 78420bf6aaa9 2024-03-29T04:52:15.535 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi099 running (22m) 47s ago 22m 488M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1f39d06fae80 2024-03-29T04:52:15.536 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi099 running (21m) 47s ago 21m 661M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6adf0209fbb4 2024-03-29T04:52:15.536 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi099 *:9095 running (14m) 47s ago 21m 64.4M - 2.43.0 a07b618ecd1d cfdb8be5d010 2024-03-29T04:52:16.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:52:16.544 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:17.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:17 smithi099 ceph-mon[94036]: from='client.45095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:17.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:17 smithi099 ceph-mon[94036]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:17.431 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:17 smithi029 ceph-mon[102350]: from='client.45095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:17.432 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:17 smithi029 ceph-mon[102350]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:17.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:17 smithi029 ceph-mon[106624]: from='client.45095 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:17.432 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:17 smithi029 ceph-mon[106624]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:18.628 INFO:teuthology.orchestra.run.smithi029.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:52:18.628 INFO:teuthology.orchestra.run.smithi029.stdout:alertmanager.a smithi029 *:9093,9094 running (15m) 13s ago 20m 22.9M - 0.25.0 c8568f914cd2 6fb7bc148154 2024-03-29T04:52:18.629 INFO:teuthology.orchestra.run.smithi029.stdout:grafana.a smithi099 *:3000 error 50s ago 20m - - 2024-03-29T04:52:18.629 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi029.gsovzu smithi029 running (19m) 13s ago 19m 19.4M - 17.2.7-789-g4213e17c cf22eb8683be e6b24be4eed9 2024-03-29T04:52:18.629 INFO:teuthology.orchestra.run.smithi029.stdout:mds.foo.smithi099.jhvkhg smithi099 running (19m) 50s ago 19m 18.8M - 17.2.7-789-g4213e17c cf22eb8683be 2d2ff55bb03a 2024-03-29T04:52:18.629 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.x smithi099 *:8443,9283,8765 running (14m) 50s ago 25m 455M - 19.0.0-1959-g8da23624 54112f57da44 1c6cc4c8d8c3 2024-03-29T04:52:18.629 INFO:teuthology.orchestra.run.smithi029.stdout:mgr.y smithi029 *:8443,9283,8765 running (15m) 13s ago 26m 510M - 19.0.0-1959-g8da23624 54112f57da44 280a22bdfead 2024-03-29T04:52:18.629 INFO:teuthology.orchestra.run.smithi029.stdout:mon.a smithi029 running (13m) 13s ago 27m 67.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 4ef356385530 2024-03-29T04:52:18.630 INFO:teuthology.orchestra.run.smithi029.stdout:mon.b smithi099 running (10m) 50s ago 25m 46.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 a53b591d5ed7 2024-03-29T04:52:18.630 INFO:teuthology.orchestra.run.smithi029.stdout:mon.c smithi029 running (11m) 13s ago 25m 51.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 69712aa77fae 2024-03-29T04:52:18.630 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.a smithi029 *:9100 running (15m) 13s ago 20m 16.6M - 1.5.0 0da6a335fe13 9026c9798e07 2024-03-29T04:52:18.630 INFO:teuthology.orchestra.run.smithi029.stdout:node-exporter.b smithi099 *:9100 running (15m) 50s ago 20m 15.5M - 1.5.0 0da6a335fe13 52f151f28461 2024-03-29T04:52:18.630 INFO:teuthology.orchestra.run.smithi029.stdout:osd.0 smithi029 running (8m) 13s ago 24m 685M 2481M 19.0.0-1959-g8da23624 54112f57da44 8456d3dfc607 2024-03-29T04:52:18.631 INFO:teuthology.orchestra.run.smithi029.stdout:osd.1 smithi029 running (5m) 13s ago 24m 249M 2481M 19.0.0-1959-g8da23624 54112f57da44 cd71767b2146 2024-03-29T04:52:18.631 INFO:teuthology.orchestra.run.smithi029.stdout:osd.2 smithi029 running (54s) 13s ago 23m 53.4M 2481M 19.0.0-1959-g8da23624 54112f57da44 7299367ca14b 2024-03-29T04:52:18.631 INFO:teuthology.orchestra.run.smithi029.stdout:osd.3 smithi029 running (18s) 13s ago 23m 13.8M 2481M 19.0.0-1959-g8da23624 54112f57da44 e0e58b4f3af9 2024-03-29T04:52:18.631 INFO:teuthology.orchestra.run.smithi029.stdout:osd.4 smithi099 running (23m) 50s ago 23m 525M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1fab66997673 2024-03-29T04:52:18.631 INFO:teuthology.orchestra.run.smithi029.stdout:osd.5 smithi099 running (22m) 50s ago 22m 983M 2480M 17.2.7-789-g4213e17c cf22eb8683be 78420bf6aaa9 2024-03-29T04:52:18.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.6 smithi099 running (22m) 50s ago 22m 488M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1f39d06fae80 2024-03-29T04:52:18.632 INFO:teuthology.orchestra.run.smithi029.stdout:osd.7 smithi099 running (21m) 50s ago 21m 661M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6adf0209fbb4 2024-03-29T04:52:18.632 INFO:teuthology.orchestra.run.smithi029.stdout:prometheus.a smithi099 *:9095 running (14m) 50s ago 21m 64.4M - 2.43.0 a07b618ecd1d cfdb8be5d010 2024-03-29T04:52:19.314 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:19 smithi029 ceph-mon[102350]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:19.314 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:19.314 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:19.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:19 smithi029 ceph-mon[106624]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:19.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:19.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:19.338 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph versions' 2024-03-29T04:52:19.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:19 smithi099 ceph-mon[94036]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:19.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:19.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:19.600 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:20.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:20 smithi099 ceph-mon[94036]: from='client.35217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:20.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:20 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:20.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:20 smithi029 ceph-mon[102350]: from='client.35217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:20.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:20 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:20.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:20 smithi029 ceph-mon[106624]: from='client.35217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:52:20.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:20 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:21.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:21 smithi099 ceph-mon[94036]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:21.422 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:21 smithi029 ceph-mon[102350]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:21.422 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:21 smithi029 ceph-mon[106624]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:21.725 INFO:teuthology.orchestra.run.smithi029.stdout:{ 2024-03-29T04:52:21.725 INFO:teuthology.orchestra.run.smithi029.stdout: "mon": { 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 3 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "mgr": { 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 2 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "osd": { 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 4, 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 4 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-03-29T04:52:21.726 INFO:teuthology.orchestra.run.smithi029.stdout: "mds": { 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 2 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout: }, 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout: "overall": { 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 6, 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 9 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout: } 2024-03-29T04:52:21.727 INFO:teuthology.orchestra.run.smithi029.stdout:} 2024-03-29T04:52:22.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:22 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/247396761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:22.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:22 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/247396761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:22.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:22 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/247396761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:52:22.510 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:22.511 INFO:teuthology.run_tasks:Running task print... 2024-03-29T04:52:22.520 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-03-29T04:52:22.520 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-29T04:52:22.546 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:52:22.546 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:52:22.546 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:52:22.812 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:23.281 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:23 smithi029 ceph-mon[102350]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:23.281 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:23 smithi029 ceph-mon[106624]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:23.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:23 smithi099 ceph-mon[94036]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:24.865 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:52:25.196 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:25 smithi029 ceph-mon[102350]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:25.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/472815369' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:52:25.197 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:52:24] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:52:25.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:25 smithi029 ceph-mon[106624]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:25.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:25 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/472815369' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:52:25.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:25 smithi099 ceph-mon[94036]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:25.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:25 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/472815369' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:52:25.576 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":41,"flags":0,"active_gid":25229,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":439211951},{"type":"v1","addr":"172.21.15.29:6801","nonce":439211951}]},"active_addr":"172.21.15.29:6801/439211951","active_change":"2024-03-29T04:38:33.745481+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25087,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.29:8443/","prometheus":"http://172.21.15.29:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":82,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":1228665397}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2848229771}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":3465912629}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.29:0","nonce":2302768086}]}]} 2024-03-29T04:52:25.581 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:52:25.581 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:52:25.581 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:52:25.839 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:26.277 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:26.278 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:26 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:26.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:26.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:26 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[102350]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[102350]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[102350]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[102350]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[106624]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[106624]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[106624]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.334 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:27 smithi029 ceph-mon[106624]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:27 smithi099 ceph-mon[94036]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:27 smithi099 ceph-mon[94036]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:27 smithi099 ceph-mon[94036]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:27 smithi099 ceph-mon[94036]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.809 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:52:27.810 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":131,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:52:08.998262+0000","last_up_change":"2024-03-29T04:52:07.991479+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:31:13.255363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:32:30.488361+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:32:30.810099+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":130,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6803","nonce":1167430522}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6805","nonce":1167430522}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6809","nonce":1167430522}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6807","nonce":1167430522}]},"public_addr":"172.21.15.29:6803/1167430522","cluster_addr":"172.21.15.29:6805/1167430522","heartbeat_back_addr":"172.21.15.29:6809/1167430522","heartbeat_front_addr":"172.21.15.29:6807/1167430522","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":110,"up_thru":130,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6811","nonce":3180340000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6813","nonce":3180340000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6817","nonce":3180340000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6815","nonce":3180340000}]},"public_addr":"172.21.15.29:6811/3180340000","cluster_addr":"172.21.15.29:6813/3180340000","heartbeat_back_addr":"172.21.15.29:6817/3180340000","heartbeat_front_addr":"172.21.15.29:6815/3180340000","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":121,"up_from":125,"up_thru":130,"down_at":122,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6819","nonce":3633025196}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6821","nonce":3633025196}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6825","nonce":3633025196}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6823","nonce":3633025196}]},"public_addr":"172.21.15.29:6819/3633025196","cluster_addr":"172.21.15.29:6821/3633025196","heartbeat_back_addr":"172.21.15.29:6825/3633025196","heartbeat_front_addr":"172.21.15.29:6823/3633025196","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":126,"up_from":130,"up_thru":130,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6827","nonce":3635564653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6829","nonce":3635564653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6833","nonce":3635564653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6831","nonce":3635564653}]},"public_addr":"172.21.15.29:6827/3635564653","cluster_addr":"172.21.15.29:6829/3635564653","heartbeat_back_addr":"172.21.15.29:6833/3635564653","heartbeat_front_addr":"172.21.15.29:6831/3635564653","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":127,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":121},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":126},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3377829364":"2024-03-30T04:35:54.810359+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/2189174236":"2024-03-30T04:38:33.745204+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1498569907":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2850164090":"2024-03-30T04:38:33.745204+0000","172.21.15.99:6834/2923568630":"2024-03-30T04:37:28.976605+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.99:0/776133218":"2024-03-30T04:37:28.976605+0000","172.21.15.29:6801/3374290780":"2024-03-30T04:35:54.810359+0000","172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3374290780":"2024-03-30T04:35:54.810359+0000","172.21.15.29:0/3191856924":"2024-03-30T04:38:33.745204+0000","172.21.15.99:0/2330824453":"2024-03-30T04:37:28.976605+0000","172.21.15.29:0/281724472":"2024-03-30T04:38:33.745204+0000","172.21.15.99:6835/2923568630":"2024-03-30T04:37:28.976605+0000","172.21.15.29:6800/3926219968":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/1956079920":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/4207961916":"2024-03-30T04:35:54.810359+0000","172.21.15.29:0/3638347893":"2024-03-30T04:35:54.810359+0000","172.21.15.99:0/2974445114":"2024-03-30T04:37:28.976605+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.99:0/3459172157":"2024-03-30T04:37:28.976605+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.29:6801/3926219968":"2024-03-30T04:38:33.745204+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/1074832730":"2024-03-30T04:35:54.810359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:52:28.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:28 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2761656123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:28.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:28 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2761656123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:28.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:28 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2761656123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:28.535 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:52:28.535 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:52:28.804 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:29.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:29 smithi029 ceph-mon[102350]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:29.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:29 smithi029 ceph-mon[106624]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:29.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:29 smithi099 ceph-mon[94036]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:30.771 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:52:30.771 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":131,"fsid":"37319ec8-ed84-11ee-95d0-87774f69a715","created":"2024-03-29T04:25:01.199792+0000","modified":"2024-03-29T04:52:08.998262+0000","last_up_change":"2024-03-29T04:52:07.991479+0000","last_in_change":"2024-03-29T04:30:19.303429+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:28:33.017273+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:31:13.255363+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:32:30.488361+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:32:30.810099+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"65937eeb-6b63-4697-8961-81cdfe926a8a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":130,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6803","nonce":1167430522}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6805","nonce":1167430522}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6808","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6809","nonce":1167430522}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":1167430522},{"type":"v1","addr":"172.21.15.29:6807","nonce":1167430522}]},"public_addr":"172.21.15.29:6803/1167430522","cluster_addr":"172.21.15.29:6805/1167430522","heartbeat_back_addr":"172.21.15.29:6809/1167430522","heartbeat_front_addr":"172.21.15.29:6807/1167430522","state":["exists","up"]},{"osd":1,"uuid":"6f30ee5e-a787-4f20-a79a-27badee86b8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":106,"up_from":110,"up_thru":130,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6810","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6811","nonce":3180340000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6812","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6813","nonce":3180340000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6816","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6817","nonce":3180340000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6814","nonce":3180340000},{"type":"v1","addr":"172.21.15.29:6815","nonce":3180340000}]},"public_addr":"172.21.15.29:6811/3180340000","cluster_addr":"172.21.15.29:6813/3180340000","heartbeat_back_addr":"172.21.15.29:6817/3180340000","heartbeat_front_addr":"172.21.15.29:6815/3180340000","state":["exists","up"]},{"osd":2,"uuid":"9112d005-33dd-4646-acea-24c19aae33ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":121,"up_from":125,"up_thru":130,"down_at":122,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6818","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6819","nonce":3633025196}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6820","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6821","nonce":3633025196}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6824","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6825","nonce":3633025196}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6822","nonce":3633025196},{"type":"v1","addr":"172.21.15.29:6823","nonce":3633025196}]},"public_addr":"172.21.15.29:6819/3633025196","cluster_addr":"172.21.15.29:6821/3633025196","heartbeat_back_addr":"172.21.15.29:6825/3633025196","heartbeat_front_addr":"172.21.15.29:6823/3633025196","state":["exists","up"]},{"osd":3,"uuid":"85f1c525-a42d-412d-843e-ef920c6ba6d2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":126,"up_from":130,"up_thru":130,"down_at":127,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6826","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6827","nonce":3635564653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6828","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6829","nonce":3635564653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6832","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6833","nonce":3635564653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6830","nonce":3635564653},{"type":"v1","addr":"172.21.15.29:6831","nonce":3635564653}]},"public_addr":"172.21.15.29:6827/3635564653","cluster_addr":"172.21.15.29:6829/3635564653","heartbeat_back_addr":"172.21.15.29:6833/3635564653","heartbeat_front_addr":"172.21.15.29:6831/3635564653","state":["exists","up"]},{"osd":4,"uuid":"f8e08e1f-4fc9-43c2-bfda-aa4b4d006013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6801","nonce":4191064052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6803","nonce":4191064052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6807","nonce":4191064052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":4191064052},{"type":"v1","addr":"172.21.15.99:6805","nonce":4191064052}]},"public_addr":"172.21.15.99:6801/4191064052","cluster_addr":"172.21.15.99:6803/4191064052","heartbeat_back_addr":"172.21.15.99:6807/4191064052","heartbeat_front_addr":"172.21.15.99:6805/4191064052","state":["exists","up"]},{"osd":5,"uuid":"1180a355-b228-4a88-9f25-23ed308ae14e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6809","nonce":3786712202}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6811","nonce":3786712202}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6815","nonce":3786712202}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3786712202},{"type":"v1","addr":"172.21.15.99:6813","nonce":3786712202}]},"public_addr":"172.21.15.99:6809/3786712202","cluster_addr":"172.21.15.99:6811/3786712202","heartbeat_back_addr":"172.21.15.99:6815/3786712202","heartbeat_front_addr":"172.21.15.99:6813/3786712202","state":["exists","up"]},{"osd":6,"uuid":"55a1eb89-3491-442f-8139-4078c86d535b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":127,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6817","nonce":2650562739}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6819","nonce":2650562739}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6823","nonce":2650562739}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2650562739},{"type":"v1","addr":"172.21.15.99:6821","nonce":2650562739}]},"public_addr":"172.21.15.99:6817/2650562739","cluster_addr":"172.21.15.99:6819/2650562739","heartbeat_back_addr":"172.21.15.99:6823/2650562739","heartbeat_front_addr":"172.21.15.99:6821/2650562739","state":["exists","up"]},{"osd":7,"uuid":"5f4c76c0-d2d8-42dd-9b72-89702f57b432","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":130,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6825","nonce":2612020879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6827","nonce":2612020879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6831","nonce":2612020879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":2612020879},{"type":"v1","addr":"172.21.15.99:6829","nonce":2612020879}]},"public_addr":"172.21.15.99:6825/2612020879","cluster_addr":"172.21.15.99:6827/2612020879","heartbeat_back_addr":"172.21.15.99:6831/2612020879","heartbeat_front_addr":"172.21.15.99:6829/2612020879","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:27:39.878214+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:05.194306+0000","dead_epoch":106},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:30.466902+0000","dead_epoch":121},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:28:56.942191+0000","dead_epoch":126},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:20.197241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:29:44.598281+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:09.588828+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:30:35.342922+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.29:0/2189174236":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/281724472":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/1956079920":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/2933871869":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/3377829364":"2024-03-30T04:35:54.810359+0000","172.21.15.29:6801/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/1498569907":"2024-03-30T04:38:33.745204+0000","172.21.15.29:6800/1886124518":"2024-03-30T04:25:49.606971+0000","172.21.15.29:0/2509984214":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2850164090":"2024-03-30T04:38:33.745204+0000","172.21.15.29:0/266316467":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/2832069104":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/4029620906":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3638694953":"2024-03-30T04:25:49.606971+0000","172.21.15.99:0/776133218":"2024-03-30T04:37:28.976605+0000","172.21.15.29:6801/3374290780":"2024-03-30T04:35:54.810359+0000","172.21.15.29:6801/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.29:6800/3163213591":"2024-03-30T04:31:09.412693+0000","172.21.15.99:0/2330824453":"2024-03-30T04:37:28.976605+0000","172.21.15.29:6800/3926219968":"2024-03-30T04:38:33.745204+0000","172.21.15.99:6835/2923568630":"2024-03-30T04:37:28.976605+0000","172.21.15.29:0/552609623":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/1742982159":"2024-03-30T04:31:09.412693+0000","172.21.15.29:0/4207961916":"2024-03-30T04:35:54.810359+0000","172.21.15.29:0/3638347893":"2024-03-30T04:35:54.810359+0000","172.21.15.99:0/2974445114":"2024-03-30T04:37:28.976605+0000","172.21.15.29:6800/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.99:0/3459172157":"2024-03-30T04:37:28.976605+0000","172.21.15.29:0/3591225422":"2024-03-30T04:25:49.606971+0000","172.21.15.99:6834/2923568630":"2024-03-30T04:37:28.976605+0000","172.21.15.29:0/4146947933":"2024-03-30T04:26:14.703329+0000","172.21.15.29:6801/3926219968":"2024-03-30T04:38:33.745204+0000","172.21.15.29:6801/2449925707":"2024-03-30T04:26:14.703329+0000","172.21.15.29:0/3191856924":"2024-03-30T04:38:33.745204+0000","172.21.15.29:6800/3374290780":"2024-03-30T04:35:54.810359+0000","172.21.15.29:0/1074832730":"2024-03-30T04:35:54.810359+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:52:31.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:31 smithi099 ceph-mon[94036]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:31.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:31 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3743408312' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:31.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:31 smithi029 ceph-mon[102350]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:31.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:31 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3743408312' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:31.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:31 smithi029 ceph-mon[106624]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:31.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:31 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3743408312' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:31.512 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:52:31.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:52:31.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:52:31.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:52:31.514 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:52:31.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:52:31.518 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:52:31.519 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:52:32.078 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.079 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.082 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.084 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.084 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.084 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.087 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.091 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:33.125 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:33 smithi029 ceph-mon[102350]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:33.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:33 smithi099 ceph-mon[94036]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:33.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:33 smithi029 ceph-mon[106624]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:34.356 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:34.356 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:34.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:34.946 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:52:34] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2024-03-29T04:52:35.284 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:35 smithi029 ceph-mon[102350]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:35.284 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:35 smithi029 ceph-mon[106624]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:35.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:35 smithi099 ceph-mon[94036]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:37.365 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:37 smithi029 ceph-mon[102350]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:37.366 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:37 smithi029 ceph-mon[106624]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:37.429 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:37 smithi099 ceph-mon[94036]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:39.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:39 smithi029 ceph-mon[102350]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:39.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:39 smithi029 ceph-mon[106624]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:39.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:39 smithi099 ceph-mon[94036]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:40.698 INFO:teuthology.orchestra.run.smithi029.stdout:558345748488 2024-03-29T04:52:40.699 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:52:41.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:41 smithi029 ceph-mon[106624]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:41.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:41 smithi029 ceph-mon[102350]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:41.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:41 smithi099 ceph-mon[94036]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:43.347 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:43 smithi029 ceph-mon[102350]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:43.348 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:43 smithi029 ceph-mon[106624]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:43.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:43 smithi099 ceph-mon[94036]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.161 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:45 smithi029 ceph-mon[102350]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.162 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:52:44] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2024-03-29T04:52:45.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:45 smithi029 ceph-mon[106624]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:45 smithi099 ceph-mon[94036]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.777 INFO:teuthology.orchestra.run.smithi029.stdout:390842024037 2024-03-29T04:52:45.777 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:52:46.632 INFO:teuthology.orchestra.run.smithi029.stdout:133143986458 2024-03-29T04:52:46.632 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:52:47.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:47 smithi029 ceph-mon[106624]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:47.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:47 smithi029 ceph-mon[102350]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:47.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:47 smithi099 ceph-mon[94036]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:47.780 INFO:teuthology.orchestra.run.smithi029.stdout:472446402625 2024-03-29T04:52:47.781 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:52:48.751 INFO:teuthology.orchestra.run.smithi029.stdout:536870912017 2024-03-29T04:52:48.751 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:52:49.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:49 smithi029 ceph-mon[102350]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:49.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:49.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:49 smithi029 ceph-mon[106624]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:49.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:49.478 INFO:teuthology.orchestra.run.smithi029.stdout:201863463186 2024-03-29T04:52:49.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:52:49.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:49 smithi099 ceph-mon[94036]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:49.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:50.396 INFO:teuthology.orchestra.run.smithi029.stdout:167503724822 2024-03-29T04:52:50.396 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:52:50.871 INFO:teuthology.orchestra.run.smithi029.stdout:236223201547 2024-03-29T04:52:50.871 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:52:50.957 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.293 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.295 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.296 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.297 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.297 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.298 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:51.437 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:51 smithi029 ceph-mon[102350]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:51.438 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:51 smithi029 ceph-mon[106624]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:51.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:51 smithi099 ceph-mon[94036]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:53.384 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:53 smithi029 ceph-mon[102350]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:53.385 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:53 smithi029 ceph-mon[106624]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:53.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:53 smithi099 ceph-mon[94036]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:54.468 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:54 smithi029 ceph-mon[102350]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:54.469 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:54 smithi029 ceph-mon[106624]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:54.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:54 smithi099 ceph-mon[94036]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:55.197 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:52:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:52:54] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2024-03-29T04:52:56.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:56 smithi099 ceph-mon[94036]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:56.696 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:56 smithi029 ceph-mon[102350]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:56.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:56 smithi029 ceph-mon[106624]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:56.751 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:58.627 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:52:58 smithi099 ceph-mon[94036]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:58.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:52:58 smithi029 ceph-mon[102350]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:58.697 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:52:58 smithi029 ceph-mon[106624]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:59.684 INFO:teuthology.orchestra.run.smithi029.stdout:201863463188 2024-03-29T04:53:00.378 INFO:teuthology.orchestra.run.smithi029.stdout:390842024040 2024-03-29T04:53:00.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:00 smithi029 ceph-mon[102350]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:00.694 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:00 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1139853395' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:53:00.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:00 smithi029 ceph-mon[106624]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:00.694 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:00 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1139853395' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:53:00.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:00 smithi099 ceph-mon[94036]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:00.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:00 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1139853395' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:53:01.520 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:01 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4059051726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:53:01.520 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:01 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4059051726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:53:01.583 INFO:teuthology.orchestra.run.smithi029.stdout:558345748492 2024-03-29T04:53:01.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:01 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4059051726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:53:01.968 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463186 got 201863463188 for osd.6 2024-03-29T04:53:01.968 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:02.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:02 smithi029 ceph-mon[102350]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:02.566 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:02 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1021942195' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:53:02.566 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:02 smithi029 ceph-mon[106624]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:02.566 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:02 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1021942195' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:53:02.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:02 smithi099 ceph-mon[94036]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:02.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:02 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1021942195' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:53:03.374 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024037 got 390842024040 for osd.0 2024-03-29T04:53:03.375 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:04.596 INFO:teuthology.orchestra.run.smithi029.stdout:472446402629 2024-03-29T04:53:04.596 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:04 smithi029 ceph-mon[102350]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:04.596 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:04.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:04 smithi029 ceph-mon[106624]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:04.597 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:04.876 INFO:tasks.cephadm.ceph_manager.ceph:need seq 558345748488 got 558345748492 for osd.3 2024-03-29T04:53:04.876 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:04.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:04 smithi099 ceph-mon[94036]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:04.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:05.145 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:53:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:53:04] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2024-03-29T04:53:05.656 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:05 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4139594256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:53:05.657 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:05 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4139594256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:53:05.860 INFO:teuthology.orchestra.run.smithi029.stdout:536870912020 2024-03-29T04:53:05.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:05 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4139594256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:53:06.555 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:06 smithi029 ceph-mon[102350]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:06.556 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:06 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4156042264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:53:06.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:06 smithi029 ceph-mon[106624]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:06.556 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:06 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4156042264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:53:06.580 INFO:tasks.cephadm.ceph_manager.ceph:need seq 536870912017 got 536870912020 for osd.2 2024-03-29T04:53:06.580 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:06.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:06 smithi099 ceph-mon[94036]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:06.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:06 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4156042264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:53:07.623 INFO:teuthology.orchestra.run.smithi029.stdout:133143986463 2024-03-29T04:53:07.827 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402625 got 472446402629 for osd.1 2024-03-29T04:53:07.827 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:07.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:07 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4147898355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:53:07.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:07 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2189838507' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:53:07.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:07 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4147898355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:53:07.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:07 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2189838507' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:53:08.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:07 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4147898355' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:53:08.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:07 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2189838507' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:53:08.200 INFO:teuthology.orchestra.run.smithi029.stdout:167503724826 2024-03-29T04:53:08.900 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:08 smithi029 ceph-mon[102350]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:08.900 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:08 smithi029 ceph-mon[106624]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:08.929 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724822 got 167503724826 for osd.5 2024-03-29T04:53:08.930 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:09.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:08 smithi099 ceph-mon[94036]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:09.565 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986458 got 133143986463 for osd.4 2024-03-29T04:53:09.565 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:10.013 INFO:teuthology.orchestra.run.smithi029.stdout:236223201552 2024-03-29T04:53:10.707 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201547 got 236223201552 for osd.7 2024-03-29T04:53:10.708 DEBUG:teuthology.parallel:result is None 2024-03-29T04:53:10.708 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:53:10.708 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:53:10.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:10 smithi029 ceph-mon[102350]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:10.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:10 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3178351248' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:53:10.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:10 smithi029 ceph-mon[106624]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:10.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:10 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3178351248' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:53:10.973 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:11.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:10 smithi099 ceph-mon[94036]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:11.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:10 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3178351248' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:53:12.824 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:12 smithi029 ceph-mon[102350]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:12.824 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:12 smithi029 ceph-mon[106624]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:13.052 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:13.053 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-03-29T04:53:13.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:12 smithi099 ceph-mon[94036]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:13.813 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":510,"stamp":"2024-03-29T04:53:11.318988+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2369920,"kb_used_data":12152,"kb_used_omap":29,"kb_used_meta":2357602,"kb_avail":747591296,"statfs":{"total":767960285184,"available":765533487104,"internally_reserved":0,"allocated":12443648,"data_stored":8199721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30045,"internal_metadata":2414185123},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003062"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":109,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.002041+0000","last_change":"2024-03-29T04:51:34.952945+0000","last_active":"2024-03-29T04:52:08.002041+0000","last_peered":"2024-03-29T04:52:08.002041+0000","last_clean":"2024-03-29T04:52:08.002041+0000","last_became_active":"2024-03-29T04:51:34.952106+0000","last_became_peered":"2024-03-29T04:51:34.952106+0000","last_unstale":"2024-03-29T04:52:08.002041+0000","last_undegraded":"2024-03-29T04:52:08.002041+0000","last_fullsized":"2024-03-29T04:52:08.002041+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:34:38.688256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016361+0000","last_change":"2024-03-29T04:52:09.016361+0000","last_active":"2024-03-29T04:52:09.016361+0000","last_peered":"2024-03-29T04:52:09.016361+0000","last_clean":"2024-03-29T04:52:09.016361+0000","last_became_active":"2024-03-29T04:52:09.016083+0000","last_became_peered":"2024-03-29T04:52:09.016083+0000","last_unstale":"2024-03-29T04:52:09.016361+0000","last_undegraded":"2024-03-29T04:52:09.016361+0000","last_fullsized":"2024-03-29T04:52:09.016361+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:06:25.084206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.011437+0000","last_change":"2024-03-29T04:52:09.011190+0000","last_active":"2024-03-29T04:52:09.011437+0000","last_peered":"2024-03-29T04:52:09.011437+0000","last_clean":"2024-03-29T04:52:09.011437+0000","last_became_active":"2024-03-29T04:52:09.010904+0000","last_became_peered":"2024-03-29T04:52:09.010904+0000","last_unstale":"2024-03-29T04:52:09.011437+0000","last_undegraded":"2024-03-29T04:52:09.011437+0000","last_fullsized":"2024-03-29T04:52:09.011437+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:30:41.574047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.021709+0000","last_change":"2024-03-29T04:52:09.019718+0000","last_active":"2024-03-29T04:52:09.021709+0000","last_peered":"2024-03-29T04:52:09.021709+0000","last_clean":"2024-03-29T04:52:09.021709+0000","last_became_active":"2024-03-29T04:52:09.018999+0000","last_became_peered":"2024-03-29T04:52:09.018999+0000","last_unstale":"2024-03-29T04:52:09.021709+0000","last_undegraded":"2024-03-29T04:52:09.021709+0000","last_fullsized":"2024-03-29T04:52:09.021709+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:04:03.536957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.015710+0000","last_change":"2024-03-29T04:52:09.015710+0000","last_active":"2024-03-29T04:52:09.015710+0000","last_peered":"2024-03-29T04:52:09.015710+0000","last_clean":"2024-03-29T04:52:09.015710+0000","last_became_active":"2024-03-29T04:52:09.013127+0000","last_became_peered":"2024-03-29T04:52:09.013127+0000","last_unstale":"2024-03-29T04:52:09.015710+0000","last_undegraded":"2024-03-29T04:52:09.015710+0000","last_fullsized":"2024-03-29T04:52:09.015710+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:08:47.048076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.005746+0000","last_change":"2024-03-29T04:47:25.724696+0000","last_active":"2024-03-29T04:52:08.005746+0000","last_peered":"2024-03-29T04:52:08.005746+0000","last_clean":"2024-03-29T04:52:08.005746+0000","last_became_active":"2024-03-29T04:47:25.724370+0000","last_became_peered":"2024-03-29T04:47:25.724370+0000","last_unstale":"2024-03-29T04:52:08.005746+0000","last_undegraded":"2024-03-29T04:52:08.005746+0000","last_fullsized":"2024-03-29T04:52:08.005746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:22:23.208797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.017616+0000","last_change":"2024-03-29T04:52:09.017616+0000","last_active":"2024-03-29T04:52:09.017616+0000","last_peered":"2024-03-29T04:52:09.017616+0000","last_clean":"2024-03-29T04:52:09.017616+0000","last_became_active":"2024-03-29T04:52:09.017302+0000","last_became_peered":"2024-03-29T04:52:09.017302+0000","last_unstale":"2024-03-29T04:52:09.017616+0000","last_undegraded":"2024-03-29T04:52:09.017616+0000","last_fullsized":"2024-03-29T04:52:09.017616+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:25:14.857689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.025006+0000","last_change":"2024-03-29T04:52:09.025006+0000","last_active":"2024-03-29T04:52:09.025006+0000","last_peered":"2024-03-29T04:52:09.025006+0000","last_clean":"2024-03-29T04:52:09.025006+0000","last_became_active":"2024-03-29T04:52:09.024627+0000","last_became_peered":"2024-03-29T04:52:09.024627+0000","last_unstale":"2024-03-29T04:52:09.025006+0000","last_undegraded":"2024-03-29T04:52:09.025006+0000","last_fullsized":"2024-03-29T04:52:09.025006+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:24:15.356402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.008158+0000","last_change":"2024-03-29T04:47:25.722749+0000","last_active":"2024-03-29T04:52:08.008158+0000","last_peered":"2024-03-29T04:52:08.008158+0000","last_clean":"2024-03-29T04:52:08.008158+0000","last_became_active":"2024-03-29T04:47:25.722369+0000","last_became_peered":"2024-03-29T04:47:25.722369+0000","last_unstale":"2024-03-29T04:52:08.008158+0000","last_undegraded":"2024-03-29T04:52:08.008158+0000","last_fullsized":"2024-03-29T04:52:08.008158+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:59:34.712929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.021972+0000","last_change":"2024-03-29T04:52:09.021972+0000","last_active":"2024-03-29T04:52:09.021972+0000","last_peered":"2024-03-29T04:52:09.021972+0000","last_clean":"2024-03-29T04:52:09.021972+0000","last_became_active":"2024-03-29T04:52:09.020505+0000","last_became_peered":"2024-03-29T04:52:09.020505+0000","last_unstale":"2024-03-29T04:52:09.021972+0000","last_undegraded":"2024-03-29T04:52:09.021972+0000","last_fullsized":"2024-03-29T04:52:09.021972+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:18:35.267374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020344+0000","last_change":"2024-03-29T04:52:09.020344+0000","last_active":"2024-03-29T04:52:09.020344+0000","last_peered":"2024-03-29T04:52:09.020344+0000","last_clean":"2024-03-29T04:52:09.020344+0000","last_became_active":"2024-03-29T04:52:09.019920+0000","last_became_peered":"2024-03-29T04:52:09.019920+0000","last_unstale":"2024-03-29T04:52:09.020344+0000","last_undegraded":"2024-03-29T04:52:09.020344+0000","last_fullsized":"2024-03-29T04:52:09.020344+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:25:56.932803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.000903+0000","last_change":"2024-03-29T04:51:34.950963+0000","last_active":"2024-03-29T04:52:09.000903+0000","last_peered":"2024-03-29T04:52:09.000903+0000","last_clean":"2024-03-29T04:52:09.000903+0000","last_became_active":"2024-03-29T04:51:34.950579+0000","last_became_peered":"2024-03-29T04:51:34.950579+0000","last_unstale":"2024-03-29T04:52:09.000903+0000","last_undegraded":"2024-03-29T04:52:09.000903+0000","last_fullsized":"2024-03-29T04:52:09.000903+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:31:53.768237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998768+0000","last_change":"2024-03-29T04:32:31.567585+0000","last_active":"2024-03-29T04:52:07.998768+0000","last_peered":"2024-03-29T04:52:07.998768+0000","last_clean":"2024-03-29T04:52:07.998768+0000","last_became_active":"2024-03-29T04:32:31.567306+0000","last_became_peered":"2024-03-29T04:32:31.567306+0000","last_unstale":"2024-03-29T04:52:07.998768+0000","last_undegraded":"2024-03-29T04:52:07.998768+0000","last_fullsized":"2024-03-29T04:52:07.998768+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:48:36.533604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998972+0000","last_change":"2024-03-29T04:47:25.726968+0000","last_active":"2024-03-29T04:52:07.998972+0000","last_peered":"2024-03-29T04:52:07.998972+0000","last_clean":"2024-03-29T04:52:07.998972+0000","last_became_active":"2024-03-29T04:47:25.725784+0000","last_became_peered":"2024-03-29T04:47:25.725784+0000","last_unstale":"2024-03-29T04:52:07.998972+0000","last_undegraded":"2024-03-29T04:52:07.998972+0000","last_fullsized":"2024-03-29T04:52:07.998972+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:16:31.119198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.002190+0000","last_change":"2024-03-29T04:51:34.952425+0000","last_active":"2024-03-29T04:52:08.002190+0000","last_peered":"2024-03-29T04:52:08.002190+0000","last_clean":"2024-03-29T04:52:08.002190+0000","last_became_active":"2024-03-29T04:51:34.952030+0000","last_became_peered":"2024-03-29T04:51:34.952030+0000","last_unstale":"2024-03-29T04:52:08.002190+0000","last_undegraded":"2024-03-29T04:52:08.002190+0000","last_fullsized":"2024-03-29T04:52:08.002190+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:38:29.844905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.022934+0000","last_change":"2024-03-29T04:52:09.022934+0000","last_active":"2024-03-29T04:52:09.022934+0000","last_peered":"2024-03-29T04:52:09.022934+0000","last_clean":"2024-03-29T04:52:09.022934+0000","last_became_active":"2024-03-29T04:52:09.022441+0000","last_became_peered":"2024-03-29T04:52:09.022441+0000","last_unstale":"2024-03-29T04:52:09.022934+0000","last_undegraded":"2024-03-29T04:52:09.022934+0000","last_fullsized":"2024-03-29T04:52:09.022934+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:13.181500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016231+0000","last_change":"2024-03-29T04:52:09.016231+0000","last_active":"2024-03-29T04:52:09.016231+0000","last_peered":"2024-03-29T04:52:09.016231+0000","last_clean":"2024-03-29T04:52:09.016231+0000","last_became_active":"2024-03-29T04:52:09.015925+0000","last_became_peered":"2024-03-29T04:52:09.015925+0000","last_unstale":"2024-03-29T04:52:09.016231+0000","last_undegraded":"2024-03-29T04:52:09.016231+0000","last_fullsized":"2024-03-29T04:52:09.016231+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:52:15.199856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":90,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019755+0000","last_change":"2024-03-29T04:52:09.019755+0000","last_active":"2024-03-29T04:52:09.019755+0000","last_peered":"2024-03-29T04:52:09.019755+0000","last_clean":"2024-03-29T04:52:09.019755+0000","last_became_active":"2024-03-29T04:52:09.019292+0000","last_became_peered":"2024-03-29T04:52:09.019292+0000","last_unstale":"2024-03-29T04:52:09.019755+0000","last_undegraded":"2024-03-29T04:52:09.019755+0000","last_fullsized":"2024-03-29T04:52:09.019755+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:38:24.832811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003048+0000","last_change":"2024-03-29T04:32:31.578200+0000","last_active":"2024-03-29T04:52:08.003048+0000","last_peered":"2024-03-29T04:52:08.003048+0000","last_clean":"2024-03-29T04:52:08.003048+0000","last_became_active":"2024-03-29T04:32:31.577900+0000","last_became_peered":"2024-03-29T04:32:31.577900+0000","last_unstale":"2024-03-29T04:52:08.003048+0000","last_undegraded":"2024-03-29T04:52:08.003048+0000","last_fullsized":"2024-03-29T04:52:08.003048+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:04:23.132474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019964+0000","last_change":"2024-03-29T04:52:09.019964+0000","last_active":"2024-03-29T04:52:09.019964+0000","last_peered":"2024-03-29T04:52:09.019964+0000","last_clean":"2024-03-29T04:52:09.019964+0000","last_became_active":"2024-03-29T04:52:09.019408+0000","last_became_peered":"2024-03-29T04:52:09.019408+0000","last_unstale":"2024-03-29T04:52:09.019964+0000","last_undegraded":"2024-03-29T04:52:09.019964+0000","last_fullsized":"2024-03-29T04:52:09.019964+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:13:57.571036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":99,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.006209+0000","last_change":"2024-03-29T04:44:25.532196+0000","last_active":"2024-03-29T04:52:08.006209+0000","last_peered":"2024-03-29T04:52:08.006209+0000","last_clean":"2024-03-29T04:52:08.006209+0000","last_became_active":"2024-03-29T04:44:25.531769+0000","last_became_peered":"2024-03-29T04:44:25.531769+0000","last_unstale":"2024-03-29T04:52:08.006209+0000","last_undegraded":"2024-03-29T04:52:08.006209+0000","last_fullsized":"2024-03-29T04:52:08.006209+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:46:52.366021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016339+0000","last_change":"2024-03-29T04:52:09.016339+0000","last_active":"2024-03-29T04:52:09.016339+0000","last_peered":"2024-03-29T04:52:09.016339+0000","last_clean":"2024-03-29T04:52:09.016339+0000","last_became_active":"2024-03-29T04:52:09.014768+0000","last_became_peered":"2024-03-29T04:52:09.014768+0000","last_unstale":"2024-03-29T04:52:09.016339+0000","last_undegraded":"2024-03-29T04:52:09.016339+0000","last_fullsized":"2024-03-29T04:52:09.016339+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:53:31.063523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":92,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.002881+0000","last_change":"2024-03-29T04:44:25.618538+0000","last_active":"2024-03-29T04:52:08.002881+0000","last_peered":"2024-03-29T04:52:08.002881+0000","last_clean":"2024-03-29T04:52:08.002881+0000","last_became_active":"2024-03-29T04:44:25.618147+0000","last_became_peered":"2024-03-29T04:44:25.618147+0000","last_unstale":"2024-03-29T04:52:08.002881+0000","last_undegraded":"2024-03-29T04:52:08.002881+0000","last_fullsized":"2024-03-29T04:52:08.002881+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:49:15.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.006908+0000","last_change":"2024-03-29T04:32:31.772869+0000","last_active":"2024-03-29T04:52:08.006908+0000","last_peered":"2024-03-29T04:52:08.006908+0000","last_clean":"2024-03-29T04:52:08.006908+0000","last_became_active":"2024-03-29T04:32:31.772518+0000","last_became_peered":"2024-03-29T04:32:31.772518+0000","last_unstale":"2024-03-29T04:52:08.006908+0000","last_undegraded":"2024-03-29T04:52:08.006908+0000","last_fullsized":"2024-03-29T04:52:08.006908+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:03:29.625832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.005488+0000","last_change":"2024-03-29T04:47:25.723173+0000","last_active":"2024-03-29T04:52:08.005488+0000","last_peered":"2024-03-29T04:52:08.005488+0000","last_clean":"2024-03-29T04:52:08.005488+0000","last_became_active":"2024-03-29T04:47:25.722281+0000","last_became_peered":"2024-03-29T04:47:25.722281+0000","last_unstale":"2024-03-29T04:52:08.005488+0000","last_undegraded":"2024-03-29T04:52:08.005488+0000","last_fullsized":"2024-03-29T04:52:08.005488+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:14:25.282710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019842+0000","last_change":"2024-03-29T04:52:09.019842+0000","last_active":"2024-03-29T04:52:09.019842+0000","last_peered":"2024-03-29T04:52:09.019842+0000","last_clean":"2024-03-29T04:52:09.019842+0000","last_became_active":"2024-03-29T04:52:09.019401+0000","last_became_peered":"2024-03-29T04:52:09.019401+0000","last_unstale":"2024-03-29T04:52:09.019842+0000","last_undegraded":"2024-03-29T04:52:09.019842+0000","last_fullsized":"2024-03-29T04:52:09.019842+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:39:14.707147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":75,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003210+0000","last_change":"2024-03-29T04:32:31.573699+0000","last_active":"2024-03-29T04:52:08.003210+0000","last_peered":"2024-03-29T04:52:08.003210+0000","last_clean":"2024-03-29T04:52:08.003210+0000","last_became_active":"2024-03-29T04:32:31.573265+0000","last_became_peered":"2024-03-29T04:32:31.573265+0000","last_unstale":"2024-03-29T04:52:08.003210+0000","last_undegraded":"2024-03-29T04:52:08.003210+0000","last_fullsized":"2024-03-29T04:52:08.003210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:05:16.290933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004801+0000","last_change":"2024-03-29T04:33:10.720781+0000","last_active":"2024-03-29T04:52:08.004801+0000","last_peered":"2024-03-29T04:52:08.004801+0000","last_clean":"2024-03-29T04:52:08.004801+0000","last_became_active":"2024-03-29T04:33:10.720378+0000","last_became_peered":"2024-03-29T04:33:10.720378+0000","last_unstale":"2024-03-29T04:52:08.004801+0000","last_undegraded":"2024-03-29T04:52:08.004801+0000","last_fullsized":"2024-03-29T04:52:08.004801+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:20:45.090317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.022467+0000","last_change":"2024-03-29T04:52:09.022467+0000","last_active":"2024-03-29T04:52:09.022467+0000","last_peered":"2024-03-29T04:52:09.022467+0000","last_clean":"2024-03-29T04:52:09.022467+0000","last_became_active":"2024-03-29T04:52:09.022100+0000","last_became_peered":"2024-03-29T04:52:09.022100+0000","last_unstale":"2024-03-29T04:52:09.022467+0000","last_undegraded":"2024-03-29T04:52:09.022467+0000","last_fullsized":"2024-03-29T04:52:09.022467+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:16:10.532449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005081+0000","last_change":"2024-03-29T04:47:25.719968+0000","last_active":"2024-03-29T04:52:09.005081+0000","last_peered":"2024-03-29T04:52:09.005081+0000","last_clean":"2024-03-29T04:52:09.005081+0000","last_became_active":"2024-03-29T04:47:25.719622+0000","last_became_peered":"2024-03-29T04:47:25.719622+0000","last_unstale":"2024-03-29T04:52:09.005081+0000","last_undegraded":"2024-03-29T04:52:09.005081+0000","last_fullsized":"2024-03-29T04:52:09.005081+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:35:06.127373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":67,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.014596+0000","last_change":"2024-03-29T04:52:09.014596+0000","last_active":"2024-03-29T04:52:09.014596+0000","last_peered":"2024-03-29T04:52:09.014596+0000","last_clean":"2024-03-29T04:52:09.014596+0000","last_became_active":"2024-03-29T04:52:09.014225+0000","last_became_peered":"2024-03-29T04:52:09.014225+0000","last_unstale":"2024-03-29T04:52:09.014596+0000","last_undegraded":"2024-03-29T04:52:09.014596+0000","last_fullsized":"2024-03-29T04:52:09.014596+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:58:05.794054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":90,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.017230+0000","last_change":"2024-03-29T04:52:09.017230+0000","last_active":"2024-03-29T04:52:09.017230+0000","last_peered":"2024-03-29T04:52:09.017230+0000","last_clean":"2024-03-29T04:52:09.017230+0000","last_became_active":"2024-03-29T04:52:09.016910+0000","last_became_peered":"2024-03-29T04:52:09.016910+0000","last_unstale":"2024-03-29T04:52:09.017230+0000","last_undegraded":"2024-03-29T04:52:09.017230+0000","last_fullsized":"2024-03-29T04:52:09.017230+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:37:24.374484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004926+0000","last_change":"2024-03-29T04:47:25.725719+0000","last_active":"2024-03-29T04:52:08.004926+0000","last_peered":"2024-03-29T04:52:08.004926+0000","last_clean":"2024-03-29T04:52:08.004926+0000","last_became_active":"2024-03-29T04:47:25.725349+0000","last_became_peered":"2024-03-29T04:47:25.725349+0000","last_unstale":"2024-03-29T04:52:08.004926+0000","last_undegraded":"2024-03-29T04:52:08.004926+0000","last_fullsized":"2024-03-29T04:52:08.004926+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:02:25.988705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.999034+0000","last_change":"2024-03-29T04:51:34.952804+0000","last_active":"2024-03-29T04:52:07.999034+0000","last_peered":"2024-03-29T04:52:07.999034+0000","last_clean":"2024-03-29T04:52:07.999034+0000","last_became_active":"2024-03-29T04:51:34.952380+0000","last_became_peered":"2024-03-29T04:51:34.952380+0000","last_unstale":"2024-03-29T04:52:07.999034+0000","last_undegraded":"2024-03-29T04:52:07.999034+0000","last_fullsized":"2024-03-29T04:52:07.999034+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:14:34.232363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998647+0000","last_change":"2024-03-29T04:51:34.952759+0000","last_active":"2024-03-29T04:52:07.998647+0000","last_peered":"2024-03-29T04:52:07.998647+0000","last_clean":"2024-03-29T04:52:07.998647+0000","last_became_active":"2024-03-29T04:51:34.952383+0000","last_became_peered":"2024-03-29T04:51:34.952383+0000","last_unstale":"2024-03-29T04:52:07.998647+0000","last_undegraded":"2024-03-29T04:52:07.998647+0000","last_fullsized":"2024-03-29T04:52:07.998647+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:18:50.989361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.999943+0000","last_change":"2024-03-29T04:47:25.723012+0000","last_active":"2024-03-29T04:52:07.999943+0000","last_peered":"2024-03-29T04:52:07.999943+0000","last_clean":"2024-03-29T04:52:07.999943+0000","last_became_active":"2024-03-29T04:47:25.722531+0000","last_became_peered":"2024-03-29T04:47:25.722531+0000","last_unstale":"2024-03-29T04:52:07.999943+0000","last_undegraded":"2024-03-29T04:52:07.999943+0000","last_fullsized":"2024-03-29T04:52:07.999943+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:49:48.883280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016474+0000","last_change":"2024-03-29T04:52:09.016474+0000","last_active":"2024-03-29T04:52:09.016474+0000","last_peered":"2024-03-29T04:52:09.016474+0000","last_clean":"2024-03-29T04:52:09.016474+0000","last_became_active":"2024-03-29T04:52:09.016113+0000","last_became_peered":"2024-03-29T04:52:09.016113+0000","last_unstale":"2024-03-29T04:52:09.016474+0000","last_undegraded":"2024-03-29T04:52:09.016474+0000","last_fullsized":"2024-03-29T04:52:09.016474+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:14:45.889964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.015663+0000","last_change":"2024-03-29T04:52:09.015663+0000","last_active":"2024-03-29T04:52:09.015663+0000","last_peered":"2024-03-29T04:52:09.015663+0000","last_clean":"2024-03-29T04:52:09.015663+0000","last_became_active":"2024-03-29T04:52:09.015320+0000","last_became_peered":"2024-03-29T04:52:09.015320+0000","last_unstale":"2024-03-29T04:52:09.015663+0000","last_undegraded":"2024-03-29T04:52:09.015663+0000","last_fullsized":"2024-03-29T04:52:09.015663+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:08:34.906457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001088+0000","last_change":"2024-03-29T04:47:25.723952+0000","last_active":"2024-03-29T04:52:09.001088+0000","last_peered":"2024-03-29T04:52:09.001088+0000","last_clean":"2024-03-29T04:52:09.001088+0000","last_became_active":"2024-03-29T04:47:25.721965+0000","last_became_peered":"2024-03-29T04:47:25.721965+0000","last_unstale":"2024-03-29T04:52:09.001088+0000","last_undegraded":"2024-03-29T04:52:09.001088+0000","last_fullsized":"2024-03-29T04:52:09.001088+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:58:56.548485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019553+0000","last_change":"2024-03-29T04:52:09.019553+0000","last_active":"2024-03-29T04:52:09.019553+0000","last_peered":"2024-03-29T04:52:09.019553+0000","last_clean":"2024-03-29T04:52:09.019553+0000","last_became_active":"2024-03-29T04:52:09.019206+0000","last_became_peered":"2024-03-29T04:52:09.019206+0000","last_unstale":"2024-03-29T04:52:09.019553+0000","last_undegraded":"2024-03-29T04:52:09.019553+0000","last_fullsized":"2024-03-29T04:52:09.019553+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:34:18.037067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.000796+0000","last_change":"2024-03-29T04:47:25.720285+0000","last_active":"2024-03-29T04:52:09.000796+0000","last_peered":"2024-03-29T04:52:09.000796+0000","last_clean":"2024-03-29T04:52:09.000796+0000","last_became_active":"2024-03-29T04:47:25.719969+0000","last_became_peered":"2024-03-29T04:47:25.719969+0000","last_unstale":"2024-03-29T04:52:09.000796+0000","last_undegraded":"2024-03-29T04:52:09.000796+0000","last_fullsized":"2024-03-29T04:52:09.000796+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:50:13.315051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.007873+0000","last_change":"2024-03-29T04:47:25.723711+0000","last_active":"2024-03-29T04:52:08.007873+0000","last_peered":"2024-03-29T04:52:08.007873+0000","last_clean":"2024-03-29T04:52:08.007873+0000","last_became_active":"2024-03-29T04:47:25.722272+0000","last_became_peered":"2024-03-29T04:47:25.722272+0000","last_unstale":"2024-03-29T04:52:08.007873+0000","last_undegraded":"2024-03-29T04:52:08.007873+0000","last_fullsized":"2024-03-29T04:52:08.007873+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:59:58.529056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.000174+0000","last_change":"2024-03-29T04:44:25.543265+0000","last_active":"2024-03-29T04:52:08.000174+0000","last_peered":"2024-03-29T04:52:08.000174+0000","last_clean":"2024-03-29T04:52:08.000174+0000","last_became_active":"2024-03-29T04:44:25.542846+0000","last_became_peered":"2024-03-29T04:44:25.542846+0000","last_unstale":"2024-03-29T04:52:08.000174+0000","last_undegraded":"2024-03-29T04:52:08.000174+0000","last_fullsized":"2024-03-29T04:52:08.000174+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:43:34.014126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.010931+0000","last_change":"2024-03-29T04:52:09.010931+0000","last_active":"2024-03-29T04:52:09.010931+0000","last_peered":"2024-03-29T04:52:09.010931+0000","last_clean":"2024-03-29T04:52:09.010931+0000","last_became_active":"2024-03-29T04:52:09.010583+0000","last_became_peered":"2024-03-29T04:52:09.010583+0000","last_unstale":"2024-03-29T04:52:09.010931+0000","last_undegraded":"2024-03-29T04:52:09.010931+0000","last_fullsized":"2024-03-29T04:52:09.010931+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:45:53.773703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":88,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.015214+0000","last_change":"2024-03-29T04:52:09.015214+0000","last_active":"2024-03-29T04:52:09.015214+0000","last_peered":"2024-03-29T04:52:09.015214+0000","last_clean":"2024-03-29T04:52:09.015214+0000","last_became_active":"2024-03-29T04:52:09.014879+0000","last_became_peered":"2024-03-29T04:52:09.014879+0000","last_unstale":"2024-03-29T04:52:09.015214+0000","last_undegraded":"2024-03-29T04:52:09.015214+0000","last_fullsized":"2024-03-29T04:52:09.015214+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:48.056047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.000938+0000","last_change":"2024-03-29T04:51:34.951211+0000","last_active":"2024-03-29T04:52:09.000938+0000","last_peered":"2024-03-29T04:52:09.000938+0000","last_clean":"2024-03-29T04:52:09.000938+0000","last_became_active":"2024-03-29T04:51:34.950018+0000","last_became_peered":"2024-03-29T04:51:34.950018+0000","last_unstale":"2024-03-29T04:52:09.000938+0000","last_undegraded":"2024-03-29T04:52:09.000938+0000","last_fullsized":"2024-03-29T04:52:09.000938+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:03:26.020521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001541+0000","last_change":"2024-03-29T04:47:25.723245+0000","last_active":"2024-03-29T04:52:09.001541+0000","last_peered":"2024-03-29T04:52:09.001541+0000","last_clean":"2024-03-29T04:52:09.001541+0000","last_became_active":"2024-03-29T04:47:25.722936+0000","last_became_peered":"2024-03-29T04:47:25.722936+0000","last_unstale":"2024-03-29T04:52:09.001541+0000","last_undegraded":"2024-03-29T04:52:09.001541+0000","last_fullsized":"2024-03-29T04:52:09.001541+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:09:29.900728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.007027+0000","last_change":"2024-03-29T04:51:34.950466+0000","last_active":"2024-03-29T04:52:08.007027+0000","last_peered":"2024-03-29T04:52:08.007027+0000","last_clean":"2024-03-29T04:52:08.007027+0000","last_became_active":"2024-03-29T04:51:34.950143+0000","last_became_peered":"2024-03-29T04:51:34.950143+0000","last_unstale":"2024-03-29T04:52:08.007027+0000","last_undegraded":"2024-03-29T04:52:08.007027+0000","last_fullsized":"2024-03-29T04:52:08.007027+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:59:39.750220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":472,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003797+0000","last_change":"2024-03-29T04:44:25.543088+0000","last_active":"2024-03-29T04:52:08.003797+0000","last_peered":"2024-03-29T04:52:08.003797+0000","last_clean":"2024-03-29T04:52:08.003797+0000","last_became_active":"2024-03-29T04:44:25.542670+0000","last_became_peered":"2024-03-29T04:44:25.542670+0000","last_unstale":"2024-03-29T04:52:08.003797+0000","last_undegraded":"2024-03-29T04:52:08.003797+0000","last_fullsized":"2024-03-29T04:52:08.003797+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_clean_scrub_stamp":"2024-03-29T04:28:33.017662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:13:36.364497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.022544+0000","last_change":"2024-03-29T04:52:09.022544+0000","last_active":"2024-03-29T04:52:09.022544+0000","last_peered":"2024-03-29T04:52:09.022544+0000","last_clean":"2024-03-29T04:52:09.022544+0000","last_became_active":"2024-03-29T04:52:09.019244+0000","last_became_peered":"2024-03-29T04:52:09.019244+0000","last_unstale":"2024-03-29T04:52:09.022544+0000","last_undegraded":"2024-03-29T04:52:09.022544+0000","last_fullsized":"2024-03-29T04:52:09.022544+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:55:56.773259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004918+0000","last_change":"2024-03-29T04:32:31.772849+0000","last_active":"2024-03-29T04:52:08.004918+0000","last_peered":"2024-03-29T04:52:08.004918+0000","last_clean":"2024-03-29T04:52:08.004918+0000","last_became_active":"2024-03-29T04:32:31.772561+0000","last_became_peered":"2024-03-29T04:32:31.772561+0000","last_unstale":"2024-03-29T04:52:08.004918+0000","last_undegraded":"2024-03-29T04:52:08.004918+0000","last_fullsized":"2024-03-29T04:52:08.004918+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:31.653522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003889+0000","last_change":"2024-03-29T04:47:25.722874+0000","last_active":"2024-03-29T04:52:08.003889+0000","last_peered":"2024-03-29T04:52:08.003889+0000","last_clean":"2024-03-29T04:52:08.003889+0000","last_became_active":"2024-03-29T04:47:25.722426+0000","last_became_peered":"2024-03-29T04:47:25.722426+0000","last_unstale":"2024-03-29T04:52:08.003889+0000","last_undegraded":"2024-03-29T04:52:08.003889+0000","last_fullsized":"2024-03-29T04:52:08.003889+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:05:38.140344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":113,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.021722+0000","last_change":"2024-03-29T04:52:09.017112+0000","last_active":"2024-03-29T04:52:09.021722+0000","last_peered":"2024-03-29T04:52:09.021722+0000","last_clean":"2024-03-29T04:52:09.021722+0000","last_became_active":"2024-03-29T04:52:09.016819+0000","last_became_peered":"2024-03-29T04:52:09.016819+0000","last_unstale":"2024-03-29T04:52:09.021722+0000","last_undegraded":"2024-03-29T04:52:09.021722+0000","last_fullsized":"2024-03-29T04:52:09.021722+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:58:41.556250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005335+0000","last_change":"2024-03-29T04:51:34.946531+0000","last_active":"2024-03-29T04:52:09.005335+0000","last_peered":"2024-03-29T04:52:09.005335+0000","last_clean":"2024-03-29T04:52:09.005335+0000","last_became_active":"2024-03-29T04:51:34.946125+0000","last_became_peered":"2024-03-29T04:51:34.946125+0000","last_unstale":"2024-03-29T04:52:09.005335+0000","last_undegraded":"2024-03-29T04:52:09.005335+0000","last_fullsized":"2024-03-29T04:52:09.005335+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:02:37.136364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003325+0000","last_change":"2024-03-29T04:44:25.610408+0000","last_active":"2024-03-29T04:52:08.003325+0000","last_peered":"2024-03-29T04:52:08.003325+0000","last_clean":"2024-03-29T04:52:08.003325+0000","last_became_active":"2024-03-29T04:44:25.610132+0000","last_became_peered":"2024-03-29T04:44:25.610132+0000","last_unstale":"2024-03-29T04:52:08.003325+0000","last_undegraded":"2024-03-29T04:52:08.003325+0000","last_fullsized":"2024-03-29T04:52:08.003325+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:58:23.459066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001202+0000","last_change":"2024-03-29T04:51:34.947138+0000","last_active":"2024-03-29T04:52:09.001202+0000","last_peered":"2024-03-29T04:52:09.001202+0000","last_clean":"2024-03-29T04:52:09.001202+0000","last_became_active":"2024-03-29T04:51:34.946790+0000","last_became_peered":"2024-03-29T04:51:34.946790+0000","last_unstale":"2024-03-29T04:52:09.001202+0000","last_undegraded":"2024-03-29T04:52:09.001202+0000","last_fullsized":"2024-03-29T04:52:09.001202+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:02:57.727641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.006590+0000","last_change":"2024-03-29T04:44:25.555175+0000","last_active":"2024-03-29T04:52:09.006590+0000","last_peered":"2024-03-29T04:52:09.006590+0000","last_clean":"2024-03-29T04:52:09.006590+0000","last_became_active":"2024-03-29T04:44:25.551060+0000","last_became_peered":"2024-03-29T04:44:25.551060+0000","last_unstale":"2024-03-29T04:52:09.006590+0000","last_undegraded":"2024-03-29T04:52:09.006590+0000","last_fullsized":"2024-03-29T04:52:09.006590+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:44:56.047380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":67,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020759+0000","last_change":"2024-03-29T04:52:09.020759+0000","last_active":"2024-03-29T04:52:09.020759+0000","last_peered":"2024-03-29T04:52:09.020759+0000","last_clean":"2024-03-29T04:52:09.020759+0000","last_became_active":"2024-03-29T04:52:09.020276+0000","last_became_peered":"2024-03-29T04:52:09.020276+0000","last_unstale":"2024-03-29T04:52:09.020759+0000","last_undegraded":"2024-03-29T04:52:09.020759+0000","last_fullsized":"2024-03-29T04:52:09.020759+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:55:53.411511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001512+0000","last_change":"2024-03-29T04:47:25.721336+0000","last_active":"2024-03-29T04:52:09.001512+0000","last_peered":"2024-03-29T04:52:09.001512+0000","last_clean":"2024-03-29T04:52:09.001512+0000","last_became_active":"2024-03-29T04:47:25.720810+0000","last_became_peered":"2024-03-29T04:47:25.720810+0000","last_unstale":"2024-03-29T04:52:09.001512+0000","last_undegraded":"2024-03-29T04:52:09.001512+0000","last_fullsized":"2024-03-29T04:52:09.001512+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:43:27.082983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.024346+0000","last_change":"2024-03-29T04:52:09.024346+0000","last_active":"2024-03-29T04:52:09.024346+0000","last_peered":"2024-03-29T04:52:09.024346+0000","last_clean":"2024-03-29T04:52:09.024346+0000","last_became_active":"2024-03-29T04:52:09.022781+0000","last_became_peered":"2024-03-29T04:52:09.022781+0000","last_unstale":"2024-03-29T04:52:09.024346+0000","last_undegraded":"2024-03-29T04:52:09.024346+0000","last_fullsized":"2024-03-29T04:52:09.024346+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:50:07.523094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004805+0000","last_change":"2024-03-29T04:51:34.955586+0000","last_active":"2024-03-29T04:52:08.004805+0000","last_peered":"2024-03-29T04:52:08.004805+0000","last_clean":"2024-03-29T04:52:08.004805+0000","last_became_active":"2024-03-29T04:51:34.955343+0000","last_became_peered":"2024-03-29T04:51:34.955343+0000","last_unstale":"2024-03-29T04:52:08.004805+0000","last_undegraded":"2024-03-29T04:52:08.004805+0000","last_fullsized":"2024-03-29T04:52:08.004805+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:31:15.457449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.006532+0000","last_change":"2024-03-29T04:47:25.726952+0000","last_active":"2024-03-29T04:52:09.006532+0000","last_peered":"2024-03-29T04:52:09.006532+0000","last_clean":"2024-03-29T04:52:09.006532+0000","last_became_active":"2024-03-29T04:47:25.723472+0000","last_became_peered":"2024-03-29T04:47:25.723472+0000","last_unstale":"2024-03-29T04:52:09.006532+0000","last_undegraded":"2024-03-29T04:52:09.006532+0000","last_fullsized":"2024-03-29T04:52:09.006532+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:20:24.208898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998728+0000","last_change":"2024-03-29T04:32:31.588347+0000","last_active":"2024-03-29T04:52:07.998728+0000","last_peered":"2024-03-29T04:52:07.998728+0000","last_clean":"2024-03-29T04:52:07.998728+0000","last_became_active":"2024-03-29T04:32:31.588080+0000","last_became_peered":"2024-03-29T04:32:31.588080+0000","last_unstale":"2024-03-29T04:52:07.998728+0000","last_undegraded":"2024-03-29T04:52:07.998728+0000","last_fullsized":"2024-03-29T04:52:07.998728+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:52:40.449275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005181+0000","last_change":"2024-03-29T04:44:25.553581+0000","last_active":"2024-03-29T04:52:09.005181+0000","last_peered":"2024-03-29T04:52:09.005181+0000","last_clean":"2024-03-29T04:52:09.005181+0000","last_became_active":"2024-03-29T04:44:25.545373+0000","last_became_peered":"2024-03-29T04:44:25.545373+0000","last_unstale":"2024-03-29T04:52:09.005181+0000","last_undegraded":"2024-03-29T04:52:09.005181+0000","last_fullsized":"2024-03-29T04:52:09.005181+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:40:36.436978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019728+0000","last_change":"2024-03-29T04:52:09.019728+0000","last_active":"2024-03-29T04:52:09.019728+0000","last_peered":"2024-03-29T04:52:09.019728+0000","last_clean":"2024-03-29T04:52:09.019728+0000","last_became_active":"2024-03-29T04:52:09.019378+0000","last_became_peered":"2024-03-29T04:52:09.019378+0000","last_unstale":"2024-03-29T04:52:09.019728+0000","last_undegraded":"2024-03-29T04:52:09.019728+0000","last_fullsized":"2024-03-29T04:52:09.019728+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:03:10.220975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.007992+0000","last_change":"2024-03-29T04:47:25.723571+0000","last_active":"2024-03-29T04:52:08.007992+0000","last_peered":"2024-03-29T04:52:08.007992+0000","last_clean":"2024-03-29T04:52:08.007992+0000","last_became_active":"2024-03-29T04:47:25.723044+0000","last_became_peered":"2024-03-29T04:47:25.723044+0000","last_unstale":"2024-03-29T04:52:08.007992+0000","last_undegraded":"2024-03-29T04:52:08.007992+0000","last_fullsized":"2024-03-29T04:52:08.007992+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:10:33.994663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020523+0000","last_change":"2024-03-29T04:52:09.020523+0000","last_active":"2024-03-29T04:52:09.020523+0000","last_peered":"2024-03-29T04:52:09.020523+0000","last_clean":"2024-03-29T04:52:09.020523+0000","last_became_active":"2024-03-29T04:52:09.020203+0000","last_became_peered":"2024-03-29T04:52:09.020203+0000","last_unstale":"2024-03-29T04:52:09.020523+0000","last_undegraded":"2024-03-29T04:52:09.020523+0000","last_fullsized":"2024-03-29T04:52:09.020523+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:52:20.001897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":76,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.008081+0000","last_change":"2024-03-29T04:32:31.572333+0000","last_active":"2024-03-29T04:52:08.008081+0000","last_peered":"2024-03-29T04:52:08.008081+0000","last_clean":"2024-03-29T04:52:08.008081+0000","last_became_active":"2024-03-29T04:32:31.572050+0000","last_became_peered":"2024-03-29T04:32:31.572050+0000","last_unstale":"2024-03-29T04:52:08.008081+0000","last_undegraded":"2024-03-29T04:52:08.008081+0000","last_fullsized":"2024-03-29T04:52:08.008081+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:41:46.118602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019784+0000","last_change":"2024-03-29T04:52:09.019784+0000","last_active":"2024-03-29T04:52:09.019784+0000","last_peered":"2024-03-29T04:52:09.019784+0000","last_clean":"2024-03-29T04:52:09.019784+0000","last_became_active":"2024-03-29T04:52:09.019408+0000","last_became_peered":"2024-03-29T04:52:09.019408+0000","last_unstale":"2024-03-29T04:52:09.019784+0000","last_undegraded":"2024-03-29T04:52:09.019784+0000","last_fullsized":"2024-03-29T04:52:09.019784+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:22:54.647345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":99,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020779+0000","last_change":"2024-03-29T04:52:09.020779+0000","last_active":"2024-03-29T04:52:09.020779+0000","last_peered":"2024-03-29T04:52:09.020779+0000","last_clean":"2024-03-29T04:52:09.020779+0000","last_became_active":"2024-03-29T04:52:09.019372+0000","last_became_peered":"2024-03-29T04:52:09.019372+0000","last_unstale":"2024-03-29T04:52:09.020779+0000","last_undegraded":"2024-03-29T04:52:09.020779+0000","last_fullsized":"2024-03-29T04:52:09.020779+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:37:28.075029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.023755+0000","last_change":"2024-03-29T04:52:09.023755+0000","last_active":"2024-03-29T04:52:09.023755+0000","last_peered":"2024-03-29T04:52:09.023755+0000","last_clean":"2024-03-29T04:52:09.023755+0000","last_became_active":"2024-03-29T04:52:09.023287+0000","last_became_peered":"2024-03-29T04:52:09.023287+0000","last_unstale":"2024-03-29T04:52:09.023755+0000","last_undegraded":"2024-03-29T04:52:09.023755+0000","last_fullsized":"2024-03-29T04:52:09.023755+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:47:32.726307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005524+0000","last_change":"2024-03-29T04:51:34.949231+0000","last_active":"2024-03-29T04:52:09.005524+0000","last_peered":"2024-03-29T04:52:09.005524+0000","last_clean":"2024-03-29T04:52:09.005524+0000","last_became_active":"2024-03-29T04:51:34.948819+0000","last_became_peered":"2024-03-29T04:51:34.948819+0000","last_unstale":"2024-03-29T04:52:09.005524+0000","last_undegraded":"2024-03-29T04:52:09.005524+0000","last_fullsized":"2024-03-29T04:52:09.005524+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:51:02.509957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998512+0000","last_change":"2024-03-29T04:47:25.723698+0000","last_active":"2024-03-29T04:52:07.998512+0000","last_peered":"2024-03-29T04:52:07.998512+0000","last_clean":"2024-03-29T04:52:07.998512+0000","last_became_active":"2024-03-29T04:47:25.723252+0000","last_became_peered":"2024-03-29T04:47:25.723252+0000","last_unstale":"2024-03-29T04:52:07.998512+0000","last_undegraded":"2024-03-29T04:52:07.998512+0000","last_fullsized":"2024-03-29T04:52:07.998512+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:46:59.721370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13602,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201553,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303308,"kb_used_data":1856,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93441844,"statfs":{"total":95995035648,"available":95684448256,"internally_reserved":0,"allocated":1900544,"data_stored":1390572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463191,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299208,"kb_used_data":1844,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445944,"statfs":{"total":95995035648,"available":95688646656,"internally_reserved":0,"allocated":1888256,"data_stored":1390484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":110,"seq":472446402631,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":204236,"kb_used_data":1344,"kb_used_omap":5,"kb_used_meta":202874,"kb_avail":93540916,"statfs":{"total":95995035648,"available":95785897984,"internally_reserved":0,"allocated":1376256,"data_stored":809847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":207743255},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":91,"seq":390842024043,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239980,"kb_used_data":2012,"kb_used_omap":5,"kb_used_meta":237946,"kb_avail":93505172,"statfs":{"total":95995035648,"available":95749296128,"internally_reserved":0,"allocated":2060288,"data_stored":1410209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6054,"internal_metadata":243656794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912021,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":74396,"kb_used_data":1292,"kb_used_omap":6,"kb_used_meta":73081,"kb_avail":93670756,"statfs":{"total":95995035648,"available":95918854144,"internally_reserved":0,"allocated":1323008,"data_stored":802848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6671,"internal_metadata":74835441},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748494,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":74768,"kb_used_data":1276,"kb_used_omap":4,"kb_used_meta":73467,"kb_avail":93670384,"statfs":{"total":95995035648,"available":95918473216,"internally_reserved":0,"allocated":1306624,"data_stored":799685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4639,"internal_metadata":75230689},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986464,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587020,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":93158132,"statfs":{"total":95995035648,"available":95393927168,"internally_reserved":0,"allocated":1302528,"data_stored":800192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724827,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587004,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":93158148,"statfs":{"total":95995035648,"available":95393943552,"internally_reserved":0,"allocated":1286144,"data_stored":795884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4350,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2935,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:53:13.816 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:53:13.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:13 smithi029 ceph-mon[102350]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:53:13.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:13 smithi029 ceph-mon[106624]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:53:14.084 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:14.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:13 smithi099 ceph-mon[94036]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:53:14.909 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:14 smithi029 ceph-mon[102350]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:14.910 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:14 smithi029 ceph-mon[106624]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:15.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:14 smithi099 ceph-mon[94036]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:15.197 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:53:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:53:14] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2024-03-29T04:53:16.121 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:16.123 INFO:teuthology.orchestra.run.smithi029.stderr:dumped all 2024-03-29T04:53:16.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:16 smithi029 ceph-mon[102350]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:16.819 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:16 smithi029 ceph-mon[102350]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:53:16.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:16 smithi029 ceph-mon[106624]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:16.820 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:16 smithi029 ceph-mon[106624]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:53:16.849 INFO:teuthology.orchestra.run.smithi029.stdout:{"pg_ready":true,"pg_map":{"version":512,"stamp":"2024-03-29T04:53:15.319948+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2369920,"kb_used_data":12152,"kb_used_omap":29,"kb_used_meta":2357602,"kb_avail":747591296,"statfs":{"total":767960285184,"available":765533487104,"internally_reserved":0,"allocated":12443648,"data_stored":8199721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30045,"internal_metadata":2414185123},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002918"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":109,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.002041+0000","last_change":"2024-03-29T04:51:34.952945+0000","last_active":"2024-03-29T04:52:08.002041+0000","last_peered":"2024-03-29T04:52:08.002041+0000","last_clean":"2024-03-29T04:52:08.002041+0000","last_became_active":"2024-03-29T04:51:34.952106+0000","last_became_peered":"2024-03-29T04:51:34.952106+0000","last_unstale":"2024-03-29T04:52:08.002041+0000","last_undegraded":"2024-03-29T04:52:08.002041+0000","last_fullsized":"2024-03-29T04:52:08.002041+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:34:38.688256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":83,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016361+0000","last_change":"2024-03-29T04:52:09.016361+0000","last_active":"2024-03-29T04:52:09.016361+0000","last_peered":"2024-03-29T04:52:09.016361+0000","last_clean":"2024-03-29T04:52:09.016361+0000","last_became_active":"2024-03-29T04:52:09.016083+0000","last_became_peered":"2024-03-29T04:52:09.016083+0000","last_unstale":"2024-03-29T04:52:09.016361+0000","last_undegraded":"2024-03-29T04:52:09.016361+0000","last_fullsized":"2024-03-29T04:52:09.016361+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:06:25.084206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.011437+0000","last_change":"2024-03-29T04:52:09.011190+0000","last_active":"2024-03-29T04:52:09.011437+0000","last_peered":"2024-03-29T04:52:09.011437+0000","last_clean":"2024-03-29T04:52:09.011437+0000","last_became_active":"2024-03-29T04:52:09.010904+0000","last_became_peered":"2024-03-29T04:52:09.010904+0000","last_unstale":"2024-03-29T04:52:09.011437+0000","last_undegraded":"2024-03-29T04:52:09.011437+0000","last_fullsized":"2024-03-29T04:52:09.011437+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:30:41.574047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.021709+0000","last_change":"2024-03-29T04:52:09.019718+0000","last_active":"2024-03-29T04:52:09.021709+0000","last_peered":"2024-03-29T04:52:09.021709+0000","last_clean":"2024-03-29T04:52:09.021709+0000","last_became_active":"2024-03-29T04:52:09.018999+0000","last_became_peered":"2024-03-29T04:52:09.018999+0000","last_unstale":"2024-03-29T04:52:09.021709+0000","last_undegraded":"2024-03-29T04:52:09.021709+0000","last_fullsized":"2024-03-29T04:52:09.021709+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:04:03.536957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.015710+0000","last_change":"2024-03-29T04:52:09.015710+0000","last_active":"2024-03-29T04:52:09.015710+0000","last_peered":"2024-03-29T04:52:09.015710+0000","last_clean":"2024-03-29T04:52:09.015710+0000","last_became_active":"2024-03-29T04:52:09.013127+0000","last_became_peered":"2024-03-29T04:52:09.013127+0000","last_unstale":"2024-03-29T04:52:09.015710+0000","last_undegraded":"2024-03-29T04:52:09.015710+0000","last_fullsized":"2024-03-29T04:52:09.015710+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:08:47.048076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.005746+0000","last_change":"2024-03-29T04:47:25.724696+0000","last_active":"2024-03-29T04:52:08.005746+0000","last_peered":"2024-03-29T04:52:08.005746+0000","last_clean":"2024-03-29T04:52:08.005746+0000","last_became_active":"2024-03-29T04:47:25.724370+0000","last_became_peered":"2024-03-29T04:47:25.724370+0000","last_unstale":"2024-03-29T04:52:08.005746+0000","last_undegraded":"2024-03-29T04:52:08.005746+0000","last_fullsized":"2024-03-29T04:52:08.005746+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:22:23.208797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.017616+0000","last_change":"2024-03-29T04:52:09.017616+0000","last_active":"2024-03-29T04:52:09.017616+0000","last_peered":"2024-03-29T04:52:09.017616+0000","last_clean":"2024-03-29T04:52:09.017616+0000","last_became_active":"2024-03-29T04:52:09.017302+0000","last_became_peered":"2024-03-29T04:52:09.017302+0000","last_unstale":"2024-03-29T04:52:09.017616+0000","last_undegraded":"2024-03-29T04:52:09.017616+0000","last_fullsized":"2024-03-29T04:52:09.017616+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:25:14.857689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.025006+0000","last_change":"2024-03-29T04:52:09.025006+0000","last_active":"2024-03-29T04:52:09.025006+0000","last_peered":"2024-03-29T04:52:09.025006+0000","last_clean":"2024-03-29T04:52:09.025006+0000","last_became_active":"2024-03-29T04:52:09.024627+0000","last_became_peered":"2024-03-29T04:52:09.024627+0000","last_unstale":"2024-03-29T04:52:09.025006+0000","last_undegraded":"2024-03-29T04:52:09.025006+0000","last_fullsized":"2024-03-29T04:52:09.025006+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:24:15.356402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.008158+0000","last_change":"2024-03-29T04:47:25.722749+0000","last_active":"2024-03-29T04:52:08.008158+0000","last_peered":"2024-03-29T04:52:08.008158+0000","last_clean":"2024-03-29T04:52:08.008158+0000","last_became_active":"2024-03-29T04:47:25.722369+0000","last_became_peered":"2024-03-29T04:47:25.722369+0000","last_unstale":"2024-03-29T04:52:08.008158+0000","last_undegraded":"2024-03-29T04:52:08.008158+0000","last_fullsized":"2024-03-29T04:52:08.008158+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:59:34.712929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.021972+0000","last_change":"2024-03-29T04:52:09.021972+0000","last_active":"2024-03-29T04:52:09.021972+0000","last_peered":"2024-03-29T04:52:09.021972+0000","last_clean":"2024-03-29T04:52:09.021972+0000","last_became_active":"2024-03-29T04:52:09.020505+0000","last_became_peered":"2024-03-29T04:52:09.020505+0000","last_unstale":"2024-03-29T04:52:09.021972+0000","last_undegraded":"2024-03-29T04:52:09.021972+0000","last_fullsized":"2024-03-29T04:52:09.021972+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:18:35.267374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020344+0000","last_change":"2024-03-29T04:52:09.020344+0000","last_active":"2024-03-29T04:52:09.020344+0000","last_peered":"2024-03-29T04:52:09.020344+0000","last_clean":"2024-03-29T04:52:09.020344+0000","last_became_active":"2024-03-29T04:52:09.019920+0000","last_became_peered":"2024-03-29T04:52:09.019920+0000","last_unstale":"2024-03-29T04:52:09.020344+0000","last_undegraded":"2024-03-29T04:52:09.020344+0000","last_fullsized":"2024-03-29T04:52:09.020344+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:25:56.932803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.000903+0000","last_change":"2024-03-29T04:51:34.950963+0000","last_active":"2024-03-29T04:52:09.000903+0000","last_peered":"2024-03-29T04:52:09.000903+0000","last_clean":"2024-03-29T04:52:09.000903+0000","last_became_active":"2024-03-29T04:51:34.950579+0000","last_became_peered":"2024-03-29T04:51:34.950579+0000","last_unstale":"2024-03-29T04:52:09.000903+0000","last_undegraded":"2024-03-29T04:52:09.000903+0000","last_fullsized":"2024-03-29T04:52:09.000903+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:31:53.768237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998768+0000","last_change":"2024-03-29T04:32:31.567585+0000","last_active":"2024-03-29T04:52:07.998768+0000","last_peered":"2024-03-29T04:52:07.998768+0000","last_clean":"2024-03-29T04:52:07.998768+0000","last_became_active":"2024-03-29T04:32:31.567306+0000","last_became_peered":"2024-03-29T04:32:31.567306+0000","last_unstale":"2024-03-29T04:52:07.998768+0000","last_undegraded":"2024-03-29T04:52:07.998768+0000","last_fullsized":"2024-03-29T04:52:07.998768+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:48:36.533604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998972+0000","last_change":"2024-03-29T04:47:25.726968+0000","last_active":"2024-03-29T04:52:07.998972+0000","last_peered":"2024-03-29T04:52:07.998972+0000","last_clean":"2024-03-29T04:52:07.998972+0000","last_became_active":"2024-03-29T04:47:25.725784+0000","last_became_peered":"2024-03-29T04:47:25.725784+0000","last_unstale":"2024-03-29T04:52:07.998972+0000","last_undegraded":"2024-03-29T04:52:07.998972+0000","last_fullsized":"2024-03-29T04:52:07.998972+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:16:31.119198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.002190+0000","last_change":"2024-03-29T04:51:34.952425+0000","last_active":"2024-03-29T04:52:08.002190+0000","last_peered":"2024-03-29T04:52:08.002190+0000","last_clean":"2024-03-29T04:52:08.002190+0000","last_became_active":"2024-03-29T04:51:34.952030+0000","last_became_peered":"2024-03-29T04:51:34.952030+0000","last_unstale":"2024-03-29T04:52:08.002190+0000","last_undegraded":"2024-03-29T04:52:08.002190+0000","last_fullsized":"2024-03-29T04:52:08.002190+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:38:29.844905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.022934+0000","last_change":"2024-03-29T04:52:09.022934+0000","last_active":"2024-03-29T04:52:09.022934+0000","last_peered":"2024-03-29T04:52:09.022934+0000","last_clean":"2024-03-29T04:52:09.022934+0000","last_became_active":"2024-03-29T04:52:09.022441+0000","last_became_peered":"2024-03-29T04:52:09.022441+0000","last_unstale":"2024-03-29T04:52:09.022934+0000","last_undegraded":"2024-03-29T04:52:09.022934+0000","last_fullsized":"2024-03-29T04:52:09.022934+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:13.181500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016231+0000","last_change":"2024-03-29T04:52:09.016231+0000","last_active":"2024-03-29T04:52:09.016231+0000","last_peered":"2024-03-29T04:52:09.016231+0000","last_clean":"2024-03-29T04:52:09.016231+0000","last_became_active":"2024-03-29T04:52:09.015925+0000","last_became_peered":"2024-03-29T04:52:09.015925+0000","last_unstale":"2024-03-29T04:52:09.016231+0000","last_undegraded":"2024-03-29T04:52:09.016231+0000","last_fullsized":"2024-03-29T04:52:09.016231+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:52:15.199856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":90,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019755+0000","last_change":"2024-03-29T04:52:09.019755+0000","last_active":"2024-03-29T04:52:09.019755+0000","last_peered":"2024-03-29T04:52:09.019755+0000","last_clean":"2024-03-29T04:52:09.019755+0000","last_became_active":"2024-03-29T04:52:09.019292+0000","last_became_peered":"2024-03-29T04:52:09.019292+0000","last_unstale":"2024-03-29T04:52:09.019755+0000","last_undegraded":"2024-03-29T04:52:09.019755+0000","last_fullsized":"2024-03-29T04:52:09.019755+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:38:24.832811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003048+0000","last_change":"2024-03-29T04:32:31.578200+0000","last_active":"2024-03-29T04:52:08.003048+0000","last_peered":"2024-03-29T04:52:08.003048+0000","last_clean":"2024-03-29T04:52:08.003048+0000","last_became_active":"2024-03-29T04:32:31.577900+0000","last_became_peered":"2024-03-29T04:32:31.577900+0000","last_unstale":"2024-03-29T04:52:08.003048+0000","last_undegraded":"2024-03-29T04:52:08.003048+0000","last_fullsized":"2024-03-29T04:52:08.003048+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:04:23.132474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019964+0000","last_change":"2024-03-29T04:52:09.019964+0000","last_active":"2024-03-29T04:52:09.019964+0000","last_peered":"2024-03-29T04:52:09.019964+0000","last_clean":"2024-03-29T04:52:09.019964+0000","last_became_active":"2024-03-29T04:52:09.019408+0000","last_became_peered":"2024-03-29T04:52:09.019408+0000","last_unstale":"2024-03-29T04:52:09.019964+0000","last_undegraded":"2024-03-29T04:52:09.019964+0000","last_fullsized":"2024-03-29T04:52:09.019964+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:13:57.571036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":99,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.006209+0000","last_change":"2024-03-29T04:44:25.532196+0000","last_active":"2024-03-29T04:52:08.006209+0000","last_peered":"2024-03-29T04:52:08.006209+0000","last_clean":"2024-03-29T04:52:08.006209+0000","last_became_active":"2024-03-29T04:44:25.531769+0000","last_became_peered":"2024-03-29T04:44:25.531769+0000","last_unstale":"2024-03-29T04:52:08.006209+0000","last_undegraded":"2024-03-29T04:52:08.006209+0000","last_fullsized":"2024-03-29T04:52:08.006209+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:46:52.366021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016339+0000","last_change":"2024-03-29T04:52:09.016339+0000","last_active":"2024-03-29T04:52:09.016339+0000","last_peered":"2024-03-29T04:52:09.016339+0000","last_clean":"2024-03-29T04:52:09.016339+0000","last_became_active":"2024-03-29T04:52:09.014768+0000","last_became_peered":"2024-03-29T04:52:09.014768+0000","last_unstale":"2024-03-29T04:52:09.016339+0000","last_undegraded":"2024-03-29T04:52:09.016339+0000","last_fullsized":"2024-03-29T04:52:09.016339+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:53:31.063523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":92,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.002881+0000","last_change":"2024-03-29T04:44:25.618538+0000","last_active":"2024-03-29T04:52:08.002881+0000","last_peered":"2024-03-29T04:52:08.002881+0000","last_clean":"2024-03-29T04:52:08.002881+0000","last_became_active":"2024-03-29T04:44:25.618147+0000","last_became_peered":"2024-03-29T04:44:25.618147+0000","last_unstale":"2024-03-29T04:52:08.002881+0000","last_undegraded":"2024-03-29T04:52:08.002881+0000","last_fullsized":"2024-03-29T04:52:08.002881+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:49:15.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.006908+0000","last_change":"2024-03-29T04:32:31.772869+0000","last_active":"2024-03-29T04:52:08.006908+0000","last_peered":"2024-03-29T04:52:08.006908+0000","last_clean":"2024-03-29T04:52:08.006908+0000","last_became_active":"2024-03-29T04:32:31.772518+0000","last_became_peered":"2024-03-29T04:32:31.772518+0000","last_unstale":"2024-03-29T04:52:08.006908+0000","last_undegraded":"2024-03-29T04:52:08.006908+0000","last_fullsized":"2024-03-29T04:52:08.006908+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:03:29.625832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.005488+0000","last_change":"2024-03-29T04:47:25.723173+0000","last_active":"2024-03-29T04:52:08.005488+0000","last_peered":"2024-03-29T04:52:08.005488+0000","last_clean":"2024-03-29T04:52:08.005488+0000","last_became_active":"2024-03-29T04:47:25.722281+0000","last_became_peered":"2024-03-29T04:47:25.722281+0000","last_unstale":"2024-03-29T04:52:08.005488+0000","last_undegraded":"2024-03-29T04:52:08.005488+0000","last_fullsized":"2024-03-29T04:52:08.005488+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:14:25.282710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019842+0000","last_change":"2024-03-29T04:52:09.019842+0000","last_active":"2024-03-29T04:52:09.019842+0000","last_peered":"2024-03-29T04:52:09.019842+0000","last_clean":"2024-03-29T04:52:09.019842+0000","last_became_active":"2024-03-29T04:52:09.019401+0000","last_became_peered":"2024-03-29T04:52:09.019401+0000","last_unstale":"2024-03-29T04:52:09.019842+0000","last_undegraded":"2024-03-29T04:52:09.019842+0000","last_fullsized":"2024-03-29T04:52:09.019842+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:39:14.707147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":75,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003210+0000","last_change":"2024-03-29T04:32:31.573699+0000","last_active":"2024-03-29T04:52:08.003210+0000","last_peered":"2024-03-29T04:52:08.003210+0000","last_clean":"2024-03-29T04:52:08.003210+0000","last_became_active":"2024-03-29T04:32:31.573265+0000","last_became_peered":"2024-03-29T04:32:31.573265+0000","last_unstale":"2024-03-29T04:52:08.003210+0000","last_undegraded":"2024-03-29T04:52:08.003210+0000","last_fullsized":"2024-03-29T04:52:08.003210+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:05:16.290933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":83,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004801+0000","last_change":"2024-03-29T04:33:10.720781+0000","last_active":"2024-03-29T04:52:08.004801+0000","last_peered":"2024-03-29T04:52:08.004801+0000","last_clean":"2024-03-29T04:52:08.004801+0000","last_became_active":"2024-03-29T04:33:10.720378+0000","last_became_peered":"2024-03-29T04:33:10.720378+0000","last_unstale":"2024-03-29T04:52:08.004801+0000","last_undegraded":"2024-03-29T04:52:08.004801+0000","last_fullsized":"2024-03-29T04:52:08.004801+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:20:45.090317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.022467+0000","last_change":"2024-03-29T04:52:09.022467+0000","last_active":"2024-03-29T04:52:09.022467+0000","last_peered":"2024-03-29T04:52:09.022467+0000","last_clean":"2024-03-29T04:52:09.022467+0000","last_became_active":"2024-03-29T04:52:09.022100+0000","last_became_peered":"2024-03-29T04:52:09.022100+0000","last_unstale":"2024-03-29T04:52:09.022467+0000","last_undegraded":"2024-03-29T04:52:09.022467+0000","last_fullsized":"2024-03-29T04:52:09.022467+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:16:10.532449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005081+0000","last_change":"2024-03-29T04:47:25.719968+0000","last_active":"2024-03-29T04:52:09.005081+0000","last_peered":"2024-03-29T04:52:09.005081+0000","last_clean":"2024-03-29T04:52:09.005081+0000","last_became_active":"2024-03-29T04:47:25.719622+0000","last_became_peered":"2024-03-29T04:47:25.719622+0000","last_unstale":"2024-03-29T04:52:09.005081+0000","last_undegraded":"2024-03-29T04:52:09.005081+0000","last_fullsized":"2024-03-29T04:52:09.005081+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:35:06.127373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":67,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.014596+0000","last_change":"2024-03-29T04:52:09.014596+0000","last_active":"2024-03-29T04:52:09.014596+0000","last_peered":"2024-03-29T04:52:09.014596+0000","last_clean":"2024-03-29T04:52:09.014596+0000","last_became_active":"2024-03-29T04:52:09.014225+0000","last_became_peered":"2024-03-29T04:52:09.014225+0000","last_unstale":"2024-03-29T04:52:09.014596+0000","last_undegraded":"2024-03-29T04:52:09.014596+0000","last_fullsized":"2024-03-29T04:52:09.014596+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:58:05.794054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":90,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.017230+0000","last_change":"2024-03-29T04:52:09.017230+0000","last_active":"2024-03-29T04:52:09.017230+0000","last_peered":"2024-03-29T04:52:09.017230+0000","last_clean":"2024-03-29T04:52:09.017230+0000","last_became_active":"2024-03-29T04:52:09.016910+0000","last_became_peered":"2024-03-29T04:52:09.016910+0000","last_unstale":"2024-03-29T04:52:09.017230+0000","last_undegraded":"2024-03-29T04:52:09.017230+0000","last_fullsized":"2024-03-29T04:52:09.017230+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:37:24.374484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004926+0000","last_change":"2024-03-29T04:47:25.725719+0000","last_active":"2024-03-29T04:52:08.004926+0000","last_peered":"2024-03-29T04:52:08.004926+0000","last_clean":"2024-03-29T04:52:08.004926+0000","last_became_active":"2024-03-29T04:47:25.725349+0000","last_became_peered":"2024-03-29T04:47:25.725349+0000","last_unstale":"2024-03-29T04:52:08.004926+0000","last_undegraded":"2024-03-29T04:52:08.004926+0000","last_fullsized":"2024-03-29T04:52:08.004926+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:02:25.988705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.999034+0000","last_change":"2024-03-29T04:51:34.952804+0000","last_active":"2024-03-29T04:52:07.999034+0000","last_peered":"2024-03-29T04:52:07.999034+0000","last_clean":"2024-03-29T04:52:07.999034+0000","last_became_active":"2024-03-29T04:51:34.952380+0000","last_became_peered":"2024-03-29T04:51:34.952380+0000","last_unstale":"2024-03-29T04:52:07.999034+0000","last_undegraded":"2024-03-29T04:52:07.999034+0000","last_fullsized":"2024-03-29T04:52:07.999034+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:14:34.232363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998647+0000","last_change":"2024-03-29T04:51:34.952759+0000","last_active":"2024-03-29T04:52:07.998647+0000","last_peered":"2024-03-29T04:52:07.998647+0000","last_clean":"2024-03-29T04:52:07.998647+0000","last_became_active":"2024-03-29T04:51:34.952383+0000","last_became_peered":"2024-03-29T04:51:34.952383+0000","last_unstale":"2024-03-29T04:52:07.998647+0000","last_undegraded":"2024-03-29T04:52:07.998647+0000","last_fullsized":"2024-03-29T04:52:07.998647+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:18:50.989361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.999943+0000","last_change":"2024-03-29T04:47:25.723012+0000","last_active":"2024-03-29T04:52:07.999943+0000","last_peered":"2024-03-29T04:52:07.999943+0000","last_clean":"2024-03-29T04:52:07.999943+0000","last_became_active":"2024-03-29T04:47:25.722531+0000","last_became_peered":"2024-03-29T04:47:25.722531+0000","last_unstale":"2024-03-29T04:52:07.999943+0000","last_undegraded":"2024-03-29T04:52:07.999943+0000","last_fullsized":"2024-03-29T04:52:07.999943+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:49:48.883280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":84,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.016474+0000","last_change":"2024-03-29T04:52:09.016474+0000","last_active":"2024-03-29T04:52:09.016474+0000","last_peered":"2024-03-29T04:52:09.016474+0000","last_clean":"2024-03-29T04:52:09.016474+0000","last_became_active":"2024-03-29T04:52:09.016113+0000","last_became_peered":"2024-03-29T04:52:09.016113+0000","last_unstale":"2024-03-29T04:52:09.016474+0000","last_undegraded":"2024-03-29T04:52:09.016474+0000","last_fullsized":"2024-03-29T04:52:09.016474+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:14:45.889964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":84,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.015663+0000","last_change":"2024-03-29T04:52:09.015663+0000","last_active":"2024-03-29T04:52:09.015663+0000","last_peered":"2024-03-29T04:52:09.015663+0000","last_clean":"2024-03-29T04:52:09.015663+0000","last_became_active":"2024-03-29T04:52:09.015320+0000","last_became_peered":"2024-03-29T04:52:09.015320+0000","last_unstale":"2024-03-29T04:52:09.015663+0000","last_undegraded":"2024-03-29T04:52:09.015663+0000","last_fullsized":"2024-03-29T04:52:09.015663+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:08:34.906457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001088+0000","last_change":"2024-03-29T04:47:25.723952+0000","last_active":"2024-03-29T04:52:09.001088+0000","last_peered":"2024-03-29T04:52:09.001088+0000","last_clean":"2024-03-29T04:52:09.001088+0000","last_became_active":"2024-03-29T04:47:25.721965+0000","last_became_peered":"2024-03-29T04:47:25.721965+0000","last_unstale":"2024-03-29T04:52:09.001088+0000","last_undegraded":"2024-03-29T04:52:09.001088+0000","last_fullsized":"2024-03-29T04:52:09.001088+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:58:56.548485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019553+0000","last_change":"2024-03-29T04:52:09.019553+0000","last_active":"2024-03-29T04:52:09.019553+0000","last_peered":"2024-03-29T04:52:09.019553+0000","last_clean":"2024-03-29T04:52:09.019553+0000","last_became_active":"2024-03-29T04:52:09.019206+0000","last_became_peered":"2024-03-29T04:52:09.019206+0000","last_unstale":"2024-03-29T04:52:09.019553+0000","last_undegraded":"2024-03-29T04:52:09.019553+0000","last_fullsized":"2024-03-29T04:52:09.019553+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:34:18.037067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.000796+0000","last_change":"2024-03-29T04:47:25.720285+0000","last_active":"2024-03-29T04:52:09.000796+0000","last_peered":"2024-03-29T04:52:09.000796+0000","last_clean":"2024-03-29T04:52:09.000796+0000","last_became_active":"2024-03-29T04:47:25.719969+0000","last_became_peered":"2024-03-29T04:47:25.719969+0000","last_unstale":"2024-03-29T04:52:09.000796+0000","last_undegraded":"2024-03-29T04:52:09.000796+0000","last_fullsized":"2024-03-29T04:52:09.000796+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:50:13.315051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.007873+0000","last_change":"2024-03-29T04:47:25.723711+0000","last_active":"2024-03-29T04:52:08.007873+0000","last_peered":"2024-03-29T04:52:08.007873+0000","last_clean":"2024-03-29T04:52:08.007873+0000","last_became_active":"2024-03-29T04:47:25.722272+0000","last_became_peered":"2024-03-29T04:47:25.722272+0000","last_unstale":"2024-03-29T04:52:08.007873+0000","last_undegraded":"2024-03-29T04:52:08.007873+0000","last_fullsized":"2024-03-29T04:52:08.007873+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:59:58.529056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.000174+0000","last_change":"2024-03-29T04:44:25.543265+0000","last_active":"2024-03-29T04:52:08.000174+0000","last_peered":"2024-03-29T04:52:08.000174+0000","last_clean":"2024-03-29T04:52:08.000174+0000","last_became_active":"2024-03-29T04:44:25.542846+0000","last_became_peered":"2024-03-29T04:44:25.542846+0000","last_unstale":"2024-03-29T04:52:08.000174+0000","last_undegraded":"2024-03-29T04:52:08.000174+0000","last_fullsized":"2024-03-29T04:52:08.000174+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:43:34.014126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.010931+0000","last_change":"2024-03-29T04:52:09.010931+0000","last_active":"2024-03-29T04:52:09.010931+0000","last_peered":"2024-03-29T04:52:09.010931+0000","last_clean":"2024-03-29T04:52:09.010931+0000","last_became_active":"2024-03-29T04:52:09.010583+0000","last_became_peered":"2024-03-29T04:52:09.010583+0000","last_unstale":"2024-03-29T04:52:09.010931+0000","last_undegraded":"2024-03-29T04:52:09.010931+0000","last_fullsized":"2024-03-29T04:52:09.010931+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:45:53.773703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":88,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.015214+0000","last_change":"2024-03-29T04:52:09.015214+0000","last_active":"2024-03-29T04:52:09.015214+0000","last_peered":"2024-03-29T04:52:09.015214+0000","last_clean":"2024-03-29T04:52:09.015214+0000","last_became_active":"2024-03-29T04:52:09.014879+0000","last_became_peered":"2024-03-29T04:52:09.014879+0000","last_unstale":"2024-03-29T04:52:09.015214+0000","last_undegraded":"2024-03-29T04:52:09.015214+0000","last_fullsized":"2024-03-29T04:52:09.015214+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:48.056047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.000938+0000","last_change":"2024-03-29T04:51:34.951211+0000","last_active":"2024-03-29T04:52:09.000938+0000","last_peered":"2024-03-29T04:52:09.000938+0000","last_clean":"2024-03-29T04:52:09.000938+0000","last_became_active":"2024-03-29T04:51:34.950018+0000","last_became_peered":"2024-03-29T04:51:34.950018+0000","last_unstale":"2024-03-29T04:52:09.000938+0000","last_undegraded":"2024-03-29T04:52:09.000938+0000","last_fullsized":"2024-03-29T04:52:09.000938+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:03:26.020521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001541+0000","last_change":"2024-03-29T04:47:25.723245+0000","last_active":"2024-03-29T04:52:09.001541+0000","last_peered":"2024-03-29T04:52:09.001541+0000","last_clean":"2024-03-29T04:52:09.001541+0000","last_became_active":"2024-03-29T04:47:25.722936+0000","last_became_peered":"2024-03-29T04:47:25.722936+0000","last_unstale":"2024-03-29T04:52:09.001541+0000","last_undegraded":"2024-03-29T04:52:09.001541+0000","last_fullsized":"2024-03-29T04:52:09.001541+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:09:29.900728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.007027+0000","last_change":"2024-03-29T04:51:34.950466+0000","last_active":"2024-03-29T04:52:08.007027+0000","last_peered":"2024-03-29T04:52:08.007027+0000","last_clean":"2024-03-29T04:52:08.007027+0000","last_became_active":"2024-03-29T04:51:34.950143+0000","last_became_peered":"2024-03-29T04:51:34.950143+0000","last_unstale":"2024-03-29T04:52:08.007027+0000","last_undegraded":"2024-03-29T04:52:08.007027+0000","last_fullsized":"2024-03-29T04:52:08.007027+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:59:39.750220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":472,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003797+0000","last_change":"2024-03-29T04:44:25.543088+0000","last_active":"2024-03-29T04:52:08.003797+0000","last_peered":"2024-03-29T04:52:08.003797+0000","last_clean":"2024-03-29T04:52:08.003797+0000","last_became_active":"2024-03-29T04:44:25.542670+0000","last_became_peered":"2024-03-29T04:44:25.542670+0000","last_unstale":"2024-03-29T04:52:08.003797+0000","last_undegraded":"2024-03-29T04:52:08.003797+0000","last_fullsized":"2024-03-29T04:52:08.003797+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:28:33.017662+0000","last_clean_scrub_stamp":"2024-03-29T04:28:33.017662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:13:36.364497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.022544+0000","last_change":"2024-03-29T04:52:09.022544+0000","last_active":"2024-03-29T04:52:09.022544+0000","last_peered":"2024-03-29T04:52:09.022544+0000","last_clean":"2024-03-29T04:52:09.022544+0000","last_became_active":"2024-03-29T04:52:09.019244+0000","last_became_peered":"2024-03-29T04:52:09.019244+0000","last_unstale":"2024-03-29T04:52:09.022544+0000","last_undegraded":"2024-03-29T04:52:09.022544+0000","last_fullsized":"2024-03-29T04:52:09.022544+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:55:56.773259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004918+0000","last_change":"2024-03-29T04:32:31.772849+0000","last_active":"2024-03-29T04:52:08.004918+0000","last_peered":"2024-03-29T04:52:08.004918+0000","last_clean":"2024-03-29T04:52:08.004918+0000","last_became_active":"2024-03-29T04:32:31.772561+0000","last_became_peered":"2024-03-29T04:32:31.772561+0000","last_unstale":"2024-03-29T04:52:08.004918+0000","last_undegraded":"2024-03-29T04:52:08.004918+0000","last_fullsized":"2024-03-29T04:52:08.004918+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:31.653522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003889+0000","last_change":"2024-03-29T04:47:25.722874+0000","last_active":"2024-03-29T04:52:08.003889+0000","last_peered":"2024-03-29T04:52:08.003889+0000","last_clean":"2024-03-29T04:52:08.003889+0000","last_became_active":"2024-03-29T04:47:25.722426+0000","last_became_peered":"2024-03-29T04:47:25.722426+0000","last_unstale":"2024-03-29T04:52:08.003889+0000","last_undegraded":"2024-03-29T04:52:08.003889+0000","last_fullsized":"2024-03-29T04:52:08.003889+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:05:38.140344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":113,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.021722+0000","last_change":"2024-03-29T04:52:09.017112+0000","last_active":"2024-03-29T04:52:09.021722+0000","last_peered":"2024-03-29T04:52:09.021722+0000","last_clean":"2024-03-29T04:52:09.021722+0000","last_became_active":"2024-03-29T04:52:09.016819+0000","last_became_peered":"2024-03-29T04:52:09.016819+0000","last_unstale":"2024-03-29T04:52:09.021722+0000","last_undegraded":"2024-03-29T04:52:09.021722+0000","last_fullsized":"2024-03-29T04:52:09.021722+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:58:41.556250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005335+0000","last_change":"2024-03-29T04:51:34.946531+0000","last_active":"2024-03-29T04:52:09.005335+0000","last_peered":"2024-03-29T04:52:09.005335+0000","last_clean":"2024-03-29T04:52:09.005335+0000","last_became_active":"2024-03-29T04:51:34.946125+0000","last_became_peered":"2024-03-29T04:51:34.946125+0000","last_unstale":"2024-03-29T04:52:09.005335+0000","last_undegraded":"2024-03-29T04:52:09.005335+0000","last_fullsized":"2024-03-29T04:52:09.005335+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:02:37.136364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.003325+0000","last_change":"2024-03-29T04:44:25.610408+0000","last_active":"2024-03-29T04:52:08.003325+0000","last_peered":"2024-03-29T04:52:08.003325+0000","last_clean":"2024-03-29T04:52:08.003325+0000","last_became_active":"2024-03-29T04:44:25.610132+0000","last_became_peered":"2024-03-29T04:44:25.610132+0000","last_unstale":"2024-03-29T04:52:08.003325+0000","last_undegraded":"2024-03-29T04:52:08.003325+0000","last_fullsized":"2024-03-29T04:52:08.003325+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:58:23.459066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":73,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001202+0000","last_change":"2024-03-29T04:51:34.947138+0000","last_active":"2024-03-29T04:52:09.001202+0000","last_peered":"2024-03-29T04:52:09.001202+0000","last_clean":"2024-03-29T04:52:09.001202+0000","last_became_active":"2024-03-29T04:51:34.946790+0000","last_became_peered":"2024-03-29T04:51:34.946790+0000","last_unstale":"2024-03-29T04:52:09.001202+0000","last_undegraded":"2024-03-29T04:52:09.001202+0000","last_fullsized":"2024-03-29T04:52:09.001202+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:02:57.727641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.006590+0000","last_change":"2024-03-29T04:44:25.555175+0000","last_active":"2024-03-29T04:52:09.006590+0000","last_peered":"2024-03-29T04:52:09.006590+0000","last_clean":"2024-03-29T04:52:09.006590+0000","last_became_active":"2024-03-29T04:44:25.551060+0000","last_became_peered":"2024-03-29T04:44:25.551060+0000","last_unstale":"2024-03-29T04:52:09.006590+0000","last_undegraded":"2024-03-29T04:52:09.006590+0000","last_fullsized":"2024-03-29T04:52:09.006590+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:44:56.047380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":67,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020759+0000","last_change":"2024-03-29T04:52:09.020759+0000","last_active":"2024-03-29T04:52:09.020759+0000","last_peered":"2024-03-29T04:52:09.020759+0000","last_clean":"2024-03-29T04:52:09.020759+0000","last_became_active":"2024-03-29T04:52:09.020276+0000","last_became_peered":"2024-03-29T04:52:09.020276+0000","last_unstale":"2024-03-29T04:52:09.020759+0000","last_undegraded":"2024-03-29T04:52:09.020759+0000","last_fullsized":"2024-03-29T04:52:09.020759+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:55:53.411511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.001512+0000","last_change":"2024-03-29T04:47:25.721336+0000","last_active":"2024-03-29T04:52:09.001512+0000","last_peered":"2024-03-29T04:52:09.001512+0000","last_clean":"2024-03-29T04:52:09.001512+0000","last_became_active":"2024-03-29T04:47:25.720810+0000","last_became_peered":"2024-03-29T04:47:25.720810+0000","last_unstale":"2024-03-29T04:52:09.001512+0000","last_undegraded":"2024-03-29T04:52:09.001512+0000","last_fullsized":"2024-03-29T04:52:09.001512+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:43:27.082983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.024346+0000","last_change":"2024-03-29T04:52:09.024346+0000","last_active":"2024-03-29T04:52:09.024346+0000","last_peered":"2024-03-29T04:52:09.024346+0000","last_clean":"2024-03-29T04:52:09.024346+0000","last_became_active":"2024-03-29T04:52:09.022781+0000","last_became_peered":"2024-03-29T04:52:09.022781+0000","last_unstale":"2024-03-29T04:52:09.024346+0000","last_undegraded":"2024-03-29T04:52:09.024346+0000","last_fullsized":"2024-03-29T04:52:09.024346+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:50:07.523094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":95,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.004805+0000","last_change":"2024-03-29T04:51:34.955586+0000","last_active":"2024-03-29T04:52:08.004805+0000","last_peered":"2024-03-29T04:52:08.004805+0000","last_clean":"2024-03-29T04:52:08.004805+0000","last_became_active":"2024-03-29T04:51:34.955343+0000","last_became_peered":"2024-03-29T04:51:34.955343+0000","last_unstale":"2024-03-29T04:52:08.004805+0000","last_undegraded":"2024-03-29T04:52:08.004805+0000","last_fullsized":"2024-03-29T04:52:08.004805+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:31:13.916080+0000","last_clean_scrub_stamp":"2024-03-29T04:31:13.916080+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:31:15.457449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.006532+0000","last_change":"2024-03-29T04:47:25.726952+0000","last_active":"2024-03-29T04:52:09.006532+0000","last_peered":"2024-03-29T04:52:09.006532+0000","last_clean":"2024-03-29T04:52:09.006532+0000","last_became_active":"2024-03-29T04:47:25.723472+0000","last_became_peered":"2024-03-29T04:47:25.723472+0000","last_unstale":"2024-03-29T04:52:09.006532+0000","last_undegraded":"2024-03-29T04:52:09.006532+0000","last_fullsized":"2024-03-29T04:52:09.006532+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:20:24.208898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998728+0000","last_change":"2024-03-29T04:32:31.588347+0000","last_active":"2024-03-29T04:52:07.998728+0000","last_peered":"2024-03-29T04:52:07.998728+0000","last_clean":"2024-03-29T04:52:07.998728+0000","last_became_active":"2024-03-29T04:32:31.588080+0000","last_became_peered":"2024-03-29T04:32:31.588080+0000","last_unstale":"2024-03-29T04:52:07.998728+0000","last_undegraded":"2024-03-29T04:52:07.998728+0000","last_fullsized":"2024-03-29T04:52:07.998728+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:52:40.449275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005181+0000","last_change":"2024-03-29T04:44:25.553581+0000","last_active":"2024-03-29T04:52:09.005181+0000","last_peered":"2024-03-29T04:52:09.005181+0000","last_clean":"2024-03-29T04:52:09.005181+0000","last_became_active":"2024-03-29T04:44:25.545373+0000","last_became_peered":"2024-03-29T04:44:25.545373+0000","last_unstale":"2024-03-29T04:52:09.005181+0000","last_undegraded":"2024-03-29T04:52:09.005181+0000","last_fullsized":"2024-03-29T04:52:09.005181+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:40:36.436978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019728+0000","last_change":"2024-03-29T04:52:09.019728+0000","last_active":"2024-03-29T04:52:09.019728+0000","last_peered":"2024-03-29T04:52:09.019728+0000","last_clean":"2024-03-29T04:52:09.019728+0000","last_became_active":"2024-03-29T04:52:09.019378+0000","last_became_peered":"2024-03-29T04:52:09.019378+0000","last_unstale":"2024-03-29T04:52:09.019728+0000","last_undegraded":"2024-03-29T04:52:09.019728+0000","last_fullsized":"2024-03-29T04:52:09.019728+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:03:10.220975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":93,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.007992+0000","last_change":"2024-03-29T04:47:25.723571+0000","last_active":"2024-03-29T04:52:08.007992+0000","last_peered":"2024-03-29T04:52:08.007992+0000","last_clean":"2024-03-29T04:52:08.007992+0000","last_became_active":"2024-03-29T04:47:25.723044+0000","last_became_peered":"2024-03-29T04:47:25.723044+0000","last_unstale":"2024-03-29T04:52:08.007992+0000","last_undegraded":"2024-03-29T04:52:08.007992+0000","last_fullsized":"2024-03-29T04:52:08.007992+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:10:33.994663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020523+0000","last_change":"2024-03-29T04:52:09.020523+0000","last_active":"2024-03-29T04:52:09.020523+0000","last_peered":"2024-03-29T04:52:09.020523+0000","last_clean":"2024-03-29T04:52:09.020523+0000","last_became_active":"2024-03-29T04:52:09.020203+0000","last_became_peered":"2024-03-29T04:52:09.020203+0000","last_unstale":"2024-03-29T04:52:09.020523+0000","last_undegraded":"2024-03-29T04:52:09.020523+0000","last_fullsized":"2024-03-29T04:52:09.020523+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:52:20.001897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":76,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:08.008081+0000","last_change":"2024-03-29T04:32:31.572333+0000","last_active":"2024-03-29T04:52:08.008081+0000","last_peered":"2024-03-29T04:52:08.008081+0000","last_clean":"2024-03-29T04:52:08.008081+0000","last_became_active":"2024-03-29T04:32:31.572050+0000","last_became_peered":"2024-03-29T04:32:31.572050+0000","last_unstale":"2024-03-29T04:52:08.008081+0000","last_undegraded":"2024-03-29T04:52:08.008081+0000","last_fullsized":"2024-03-29T04:52:08.008081+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:41:46.118602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.019784+0000","last_change":"2024-03-29T04:52:09.019784+0000","last_active":"2024-03-29T04:52:09.019784+0000","last_peered":"2024-03-29T04:52:09.019784+0000","last_clean":"2024-03-29T04:52:09.019784+0000","last_became_active":"2024-03-29T04:52:09.019408+0000","last_became_peered":"2024-03-29T04:52:09.019408+0000","last_unstale":"2024-03-29T04:52:09.019784+0000","last_undegraded":"2024-03-29T04:52:09.019784+0000","last_fullsized":"2024-03-29T04:52:09.019784+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:22:54.647345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":99,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.020779+0000","last_change":"2024-03-29T04:52:09.020779+0000","last_active":"2024-03-29T04:52:09.020779+0000","last_peered":"2024-03-29T04:52:09.020779+0000","last_clean":"2024-03-29T04:52:09.020779+0000","last_became_active":"2024-03-29T04:52:09.019372+0000","last_became_peered":"2024-03-29T04:52:09.019372+0000","last_unstale":"2024-03-29T04:52:09.020779+0000","last_undegraded":"2024-03-29T04:52:09.020779+0000","last_fullsized":"2024-03-29T04:52:09.020779+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:37:28.075029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.023755+0000","last_change":"2024-03-29T04:52:09.023755+0000","last_active":"2024-03-29T04:52:09.023755+0000","last_peered":"2024-03-29T04:52:09.023755+0000","last_clean":"2024-03-29T04:52:09.023755+0000","last_became_active":"2024-03-29T04:52:09.023287+0000","last_became_peered":"2024-03-29T04:52:09.023287+0000","last_unstale":"2024-03-29T04:52:09.023755+0000","last_undegraded":"2024-03-29T04:52:09.023755+0000","last_fullsized":"2024-03-29T04:52:09.023755+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:47:32.726307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":116,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-03-29T04:52:09.005524+0000","last_change":"2024-03-29T04:51:34.949231+0000","last_active":"2024-03-29T04:52:09.005524+0000","last_peered":"2024-03-29T04:52:09.005524+0000","last_clean":"2024-03-29T04:52:09.005524+0000","last_became_active":"2024-03-29T04:51:34.948819+0000","last_became_peered":"2024-03-29T04:51:34.948819+0000","last_unstale":"2024-03-29T04:52:09.005524+0000","last_undegraded":"2024-03-29T04:52:09.005524+0000","last_fullsized":"2024-03-29T04:52:09.005524+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:30.545601+0000","last_clean_scrub_stamp":"2024-03-29T04:32:30.545601+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:51:02.509957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2024-03-29T04:52:07.998512+0000","last_change":"2024-03-29T04:47:25.723698+0000","last_active":"2024-03-29T04:52:07.998512+0000","last_peered":"2024-03-29T04:52:07.998512+0000","last_clean":"2024-03-29T04:52:07.998512+0000","last_became_active":"2024-03-29T04:47:25.723252+0000","last_became_peered":"2024-03-29T04:47:25.723252+0000","last_unstale":"2024-03-29T04:52:07.998512+0000","last_undegraded":"2024-03-29T04:52:07.998512+0000","last_fullsized":"2024-03-29T04:52:07.998512+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:32:31.552581+0000","last_clean_scrub_stamp":"2024-03-29T04:32:31.552581+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:46:59.721370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13602,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201554,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303308,"kb_used_data":1856,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93441844,"statfs":{"total":95995035648,"available":95684448256,"internally_reserved":0,"allocated":1900544,"data_stored":1390572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463192,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299208,"kb_used_data":1844,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445944,"statfs":{"total":95995035648,"available":95688646656,"internally_reserved":0,"allocated":1888256,"data_stored":1390484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":110,"seq":472446402632,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":204236,"kb_used_data":1344,"kb_used_omap":5,"kb_used_meta":202874,"kb_avail":93540916,"statfs":{"total":95995035648,"available":95785897984,"internally_reserved":0,"allocated":1376256,"data_stored":809847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":207743255},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":91,"seq":390842024044,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":239980,"kb_used_data":2012,"kb_used_omap":5,"kb_used_meta":237946,"kb_avail":93505172,"statfs":{"total":95995035648,"available":95749296128,"internally_reserved":0,"allocated":2060288,"data_stored":1410209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6054,"internal_metadata":243656794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":125,"seq":536870912022,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":74396,"kb_used_data":1292,"kb_used_omap":6,"kb_used_meta":73081,"kb_avail":93670756,"statfs":{"total":95995035648,"available":95918854144,"internally_reserved":0,"allocated":1323008,"data_stored":802848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6671,"internal_metadata":74835441},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":130,"seq":558345748495,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":74768,"kb_used_data":1276,"kb_used_omap":4,"kb_used_meta":73467,"kb_avail":93670384,"statfs":{"total":95995035648,"available":95918473216,"internally_reserved":0,"allocated":1306624,"data_stored":799685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4639,"internal_metadata":75230689},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986465,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587020,"kb_used_data":1272,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":93158132,"statfs":{"total":95995035648,"available":95393927168,"internally_reserved":0,"allocated":1302528,"data_stored":800192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724828,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587004,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":93158148,"statfs":{"total":95995035648,"available":95393943552,"internally_reserved":0,"allocated":1286144,"data_stored":795884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4350,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2935,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:53:16.852 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:53:16.852 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:53:16.853 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:53:16.853 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:17.124 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:17.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:16 smithi099 ceph-mon[94036]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:17.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:16 smithi099 ceph-mon[94036]: from='client.35349 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:53:18.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:18 smithi029 ceph-mon[102350]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:18.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:18 smithi029 ceph-mon[106624]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:19.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:18 smithi099 ceph-mon[94036]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:19.154 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:19.155 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:19.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:19.863 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:19 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3383994526' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:19.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:19.863 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:19 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3383994526' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:20.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:19 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3383994526' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:20.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:20 smithi029 ceph-mon[102350]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:20.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:20 smithi029 ceph-mon[106624]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:21.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:20 smithi099 ceph-mon[94036]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:22.889 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:22.920 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:22 smithi029 ceph-mon[102350]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:22.921 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:22 smithi029 ceph-mon[106624]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:23.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:22 smithi099 ceph-mon[94036]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:23.163 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:24.946 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:24 smithi029 ceph-mon[102350]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:24.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:24 smithi029 ceph-mon[106624]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:24.947 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:53:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:53:24] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2024-03-29T04:53:25.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:24 smithi099 ceph-mon[94036]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:25.216 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:25.216 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:25.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:25 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3186638126' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:25.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:25 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:25 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3186638126' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:25.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:25 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:26.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:25 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3186638126' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:26.127 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:25 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:26.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:26 smithi029 ceph-mon[102350]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:26.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:26 smithi029 ceph-mon[106624]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:27.059 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:26 smithi099 ceph-mon[94036]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:28.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:28 smithi029 ceph-mon[102350]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:28.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:28 smithi029 ceph-mon[106624]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:28.975 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:28 smithi099 ceph-mon[94036]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:28.983 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:29.597 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:30.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[102350]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:30.814 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.815 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.815 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[106624]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:30.815 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.816 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:30 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:30 smithi099 ceph-mon[94036]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:30.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:30 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:30 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:30 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:30.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:30 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:31.669 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:31.669 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:31.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:31.669 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[102350]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[102350]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[106624]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[106624]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.670 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:31 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:31.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:31 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:31.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:31 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:31.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:31 smithi099 ceph-mon[94036]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:31 smithi099 ceph-mon[94036]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:31 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:53:32.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:32 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/662141618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:32.698 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:32 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/662141618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:32.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:32 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/662141618' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:33.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:33 smithi099 ceph-mon[94036]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:33.947 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:33 smithi029 ceph-mon[102350]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:33.947 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:33 smithi029 ceph-mon[106624]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:34.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:34.910 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:34.911 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:35.197 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:53:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:53:34] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:53:35.325 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:35.601 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:35.697 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:35 smithi029 ceph-mon[102350]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:35.862 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:35 smithi029 ceph-mon[106624]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:35.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:35 smithi099 ceph-mon[94036]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:37.648 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:37.648 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:38.234 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:38 smithi029 ceph-mon[102350]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:38.235 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3535568596' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:38.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:38 smithi029 ceph-mon[106624]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:38.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3535568596' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:38.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:38 smithi099 ceph-mon[94036]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:38.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3535568596' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:40.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:40 smithi099 ceph-mon[94036]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:40.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:40 smithi029 ceph-mon[102350]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:40.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:40 smithi029 ceph-mon[106624]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:41.261 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:41.536 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:42.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:42 smithi099 ceph-mon[94036]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:42.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:42 smithi029 ceph-mon[102350]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:42.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:42 smithi029 ceph-mon[106624]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:43.698 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:43.699 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:44.326 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:44 smithi029 ceph-mon[102350]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:44.326 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:44 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1477060309' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:44.327 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:44 smithi029 ceph-mon[106624]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:44.327 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:44 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1477060309' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:44.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:44 smithi099 ceph-mon[94036]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:44.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:44 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1477060309' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:45.197 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:53:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:53:44] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:53:46.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:46 smithi099 ceph-mon[94036]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:46.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:46 smithi029 ceph-mon[102350]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:46.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:46 smithi029 ceph-mon[106624]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:47.356 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:47.631 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:48.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:48 smithi099 ceph-mon[94036]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:48.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:48 smithi029 ceph-mon[102350]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:48.447 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:48 smithi029 ceph-mon[106624]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:49.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:49.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:49.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:49.753 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:49.753 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:50.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:50 smithi099 ceph-mon[94036]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:50.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:50 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3337116937' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:50 smithi029 ceph-mon[102350]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:50.421 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:50 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3337116937' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:50 smithi029 ceph-mon[106624]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:50.421 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:50 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3337116937' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:52.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:52 smithi099 ceph-mon[94036]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:52.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:52 smithi029 ceph-mon[102350]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:52.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:52 smithi029 ceph-mon[106624]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:53.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:53.724 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:54.162 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:54 smithi029 ceph-mon[102350]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:54.162 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:54 smithi029 ceph-mon[106624]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:54.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:54 smithi099 ceph-mon[94036]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:55.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:53:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:53:54] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.43.0" 2024-03-29T04:53:55.825 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:53:55.826 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:56 smithi029 ceph-mon[102350]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:56.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:56 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/868509318' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:56 smithi029 ceph-mon[106624]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:56.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:56 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/868509318' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:56.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:56 smithi099 ceph-mon[94036]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:56.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:56 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/868509318' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:58.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:53:58 smithi099 ceph-mon[94036]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:58.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:53:58 smithi029 ceph-mon[102350]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:58.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:53:58 smithi029 ceph-mon[106624]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:59.568 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:59.844 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:00.291 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:00 smithi029 ceph-mon[102350]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:00.291 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:00 smithi029 ceph-mon[106624]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:00.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:00 smithi099 ceph-mon[94036]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:01.935 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:01.935 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:02.197 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:02 smithi029 ceph-mon[106624]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:02.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:02 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1065731693' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:02.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:02 smithi029 ceph-mon[102350]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:02.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:02 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1065731693' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:02.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:02 smithi099 ceph-mon[94036]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:02.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:02 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1065731693' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:04.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:04 smithi099 ceph-mon[94036]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:04.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:04.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:04 smithi029 ceph-mon[102350]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:04.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:04.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:04 smithi029 ceph-mon[106624]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:04.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:05.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:54:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:54:04] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.43.0" 2024-03-29T04:54:05.746 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:06.021 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:06.328 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:06 smithi029 ceph-mon[102350]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:06.328 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:06 smithi029 ceph-mon[106624]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:06.377 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:06 smithi099 ceph-mon[94036]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:08.113 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:08.113 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:08.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:08 smithi029 ceph-mon[102350]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:08.114 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:08 smithi029 ceph-mon[106624]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:08.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:08 smithi099 ceph-mon[94036]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:09.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/859994971' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:09.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/859994971' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:09.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/859994971' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:10.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:10 smithi099 ceph-mon[94036]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:10.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:10 smithi029 ceph-mon[102350]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:10.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:10 smithi029 ceph-mon[106624]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:11.858 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:12.133 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:12.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:12 smithi029 ceph-mon[102350]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:12.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:12 smithi029 ceph-mon[106624]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:12.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:12 smithi099 ceph-mon[94036]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:14.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:14 smithi029 ceph-mon[102350]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:14.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:14 smithi029 ceph-mon[106624]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:14.202 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:14.203 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:14.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:14 smithi099 ceph-mon[94036]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:15.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/797746923' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:15.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/797746923' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:15.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:54:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:54:14] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.43.0" 2024-03-29T04:54:15.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/797746923' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:16.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:16 smithi099 ceph-mon[94036]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:16.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:16 smithi029 ceph-mon[102350]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:16.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:16 smithi029 ceph-mon[106624]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:17.882 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:18.158 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:18.197 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:18 smithi029 ceph-mon[102350]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:18.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:18 smithi029 ceph-mon[106624]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:18.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:18 smithi099 ceph-mon[94036]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:19.330 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:19.330 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:19.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:20.091 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:20.091 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:20.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:20 smithi099 ceph-mon[94036]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:20.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:20 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2028042986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:20.445 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:20 smithi029 ceph-mon[102350]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:20.446 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:20 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2028042986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:20.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:20 smithi029 ceph-mon[106624]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:20.446 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:20 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2028042986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:22.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:22 smithi099 ceph-mon[94036]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:22.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:22 smithi029 ceph-mon[102350]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:22.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:22 smithi029 ceph-mon[106624]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:23.795 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:24.069 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:24.365 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:24 smithi029 ceph-mon[102350]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:24.365 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:24 smithi029 ceph-mon[106624]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:24.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:24 smithi099 ceph-mon[94036]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:25.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:54:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:54:24] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:54:26.137 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:26.137 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:26.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:26 smithi099 ceph-mon[94036]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:26.430 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:26 smithi029 ceph-mon[102350]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:26.430 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:26 smithi029 ceph-mon[106624]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:27.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:27 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1136879225' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:27.400 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:27 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1136879225' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:27.400 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:27 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1136879225' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:28.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:28 smithi099 ceph-mon[94036]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:28.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:28 smithi029 ceph-mon[102350]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:28.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:28 smithi029 ceph-mon[106624]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:29.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:30.104 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:30.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:30 smithi099 ceph-mon[94036]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:30.387 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:30 smithi029 ceph-mon[102350]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:30.387 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:30 smithi029 ceph-mon[106624]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:31.328 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:31 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:31.328 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:31 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:31.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:31 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:32.166 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:32.166 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:32.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:32 smithi099 ceph-mon[94036]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:32.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:32 smithi029 ceph-mon[102350]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:32.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:32 smithi029 ceph-mon[106624]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:33.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3026241854' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:33.447 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3026241854' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:33.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3026241854' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:34.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:34 smithi029 ceph-mon[102350]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:34.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:34.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:34 smithi029 ceph-mon[106624]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:34.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:34.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:34 smithi099 ceph-mon[94036]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:34.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:34.948 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:54:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:54:34] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:54:35.848 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:36.125 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:36.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[102350]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:36.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.135 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.136 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.390 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[106624]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:36.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.391 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:36 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:36 smithi099 ceph-mon[94036]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:36.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:36 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:36 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:36 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:36.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:36 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:37.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[102350]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:37.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:37.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:37.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[102350]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:37.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[102350]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:37.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:37.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[106624]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:37.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:37.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:37.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[106624]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:37.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[106624]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:37.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:37 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:38.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:37 smithi099 ceph-mon[94036]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:38.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:38.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:38.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:37 smithi099 ceph-mon[94036]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:38.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:37 smithi099 ceph-mon[94036]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:38.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:37 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:54:38.151 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:38.151 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:38.833 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3418634625' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:38.834 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3418634625' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:38.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3418634625' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:39.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:39 smithi029 ceph-mon[102350]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:39.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:39 smithi029 ceph-mon[106624]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:40.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:39 smithi099 ceph-mon[94036]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:41.864 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:42.140 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:42.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:42 smithi029 ceph-mon[102350]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:42.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:42 smithi029 ceph-mon[106624]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:42.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:42 smithi099 ceph-mon[94036]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:44.176 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:44.176 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:44.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:44 smithi099 ceph-mon[94036]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:44.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:44 smithi029 ceph-mon[102350]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:44.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:44 smithi029 ceph-mon[106624]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:45.112 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:54:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:54:44] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:54:45.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:45 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/465094242' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:45.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:45 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/465094242' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:45.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:45 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/465094242' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:46.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:46 smithi099 ceph-mon[94036]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:46.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:46 smithi029 ceph-mon[102350]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:46.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:46 smithi029 ceph-mon[106624]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:47.854 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:48.128 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:48.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:48 smithi029 ceph-mon[102350]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:48.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:48 smithi029 ceph-mon[106624]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:48.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:48 smithi099 ceph-mon[94036]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:49.277 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:49.278 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:49.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:50.096 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:50.096 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:50.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:50 smithi099 ceph-mon[94036]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:50.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:50 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/188947918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:50.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:50 smithi029 ceph-mon[102350]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:50.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:50 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/188947918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:50.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:50 smithi029 ceph-mon[106624]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:50.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:50 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/188947918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:52.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:52 smithi099 ceph-mon[94036]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:52.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:52 smithi029 ceph-mon[102350]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:52.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:52 smithi029 ceph-mon[106624]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:53.827 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:54.102 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:54.377 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:54 smithi029 ceph-mon[102350]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:54.378 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:54 smithi029 ceph-mon[106624]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:54.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:54 smithi099 ceph-mon[94036]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:55.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:54:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:54:54] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2024-03-29T04:54:56.212 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:54:56.212 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:56.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:56 smithi029 ceph-mon[102350]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:56.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:56 smithi029 ceph-mon[106624]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:56.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:56 smithi099 ceph-mon[94036]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:57.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:57 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3370837368' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:57.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:57 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3370837368' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:57.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:57 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3370837368' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:58.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:54:58 smithi029 ceph-mon[102350]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:58.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:54:58 smithi029 ceph-mon[106624]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:58.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:54:58 smithi099 ceph-mon[94036]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:54:59.999 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:00.280 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:00.402 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:00 smithi029 ceph-mon[102350]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:00.402 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:00 smithi029 ceph-mon[106624]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:00.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:00 smithi099 ceph-mon[94036]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:02.338 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:02.338 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:02.351 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:02 smithi029 ceph-mon[102350]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:02.351 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:02 smithi029 ceph-mon[106624]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:02.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:02 smithi099 ceph-mon[94036]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:03.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3513738467' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:03.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3513738467' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:03.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3513738467' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:04.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:04 smithi029 ceph-mon[102350]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:04.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:04.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:04 smithi029 ceph-mon[106624]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:04.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:04.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:04 smithi099 ceph-mon[94036]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:04.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:05.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:55:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:55:04] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2024-03-29T04:55:05.938 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:06.213 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:06.335 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:06 smithi029 ceph-mon[102350]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:06.336 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:06 smithi029 ceph-mon[106624]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:06.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:06 smithi099 ceph-mon[94036]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:08.280 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:08.280 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:08.280 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:08 smithi029 ceph-mon[102350]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:08.280 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:08 smithi029 ceph-mon[106624]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:08.430 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:08 smithi099 ceph-mon[94036]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:09.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2843494322' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:09.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2843494322' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:09.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2843494322' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:10.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:10 smithi029 ceph-mon[102350]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:10.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:10 smithi029 ceph-mon[106624]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:10.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:10 smithi099 ceph-mon[94036]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:11.961 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:12.237 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:12.359 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:12 smithi029 ceph-mon[102350]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:12.360 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:12 smithi029 ceph-mon[106624]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:12.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:12 smithi099 ceph-mon[94036]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:14.228 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:14.228 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:14.229 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:14 smithi029 ceph-mon[102350]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:14.229 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:14 smithi029 ceph-mon[106624]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:14.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:14 smithi099 ceph-mon[94036]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:15.166 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:55:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:55:14] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2024-03-29T04:55:15.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2821044283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:15.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2821044283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:15.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2821044283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:16.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:16 smithi029 ceph-mon[102350]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:16.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:16 smithi029 ceph-mon[106624]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:16.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:16 smithi099 ceph-mon[94036]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:17.962 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:18.238 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:18.361 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:18 smithi029 ceph-mon[102350]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:18.361 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:18 smithi029 ceph-mon[106624]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:18.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:18 smithi099 ceph-mon[94036]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:19.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:19.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:19.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:20.330 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:20.331 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:20.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:20 smithi029 ceph-mon[102350]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:20.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:20 smithi029 ceph-mon[106624]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:20.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:20 smithi099 ceph-mon[94036]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:21.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1620630186' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:21.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:21 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1620630186' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:21.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:21 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1620630186' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:22.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:22 smithi029 ceph-mon[102350]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:22.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:22 smithi029 ceph-mon[106624]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:22.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:22 smithi099 ceph-mon[94036]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:23.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:23 smithi029 ceph-mon[106624]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:23.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:23 smithi029 ceph-mon[102350]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:23.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:23 smithi099 ceph-mon[94036]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:24.225 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:24.499 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:25.201 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:55:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:55:24] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:55:26.255 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:26 smithi029 ceph-mon[102350]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:26.255 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:26 smithi029 ceph-mon[106624]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:26.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:26 smithi099 ceph-mon[94036]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:26.527 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:26.527 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:27.164 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:27 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1379476753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:27.164 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:27 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1379476753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:27.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:27 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1379476753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:28.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:28 smithi099 ceph-mon[94036]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:28.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:28 smithi029 ceph-mon[102350]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:28.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:28 smithi029 ceph-mon[106624]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:30.190 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:30.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:30 smithi099 ceph-mon[94036]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:30.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:30 smithi029 ceph-mon[102350]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:30.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:30 smithi029 ceph-mon[106624]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:30.469 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:32.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:32 smithi029 ceph-mon[102350]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:32.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:32 smithi029 ceph-mon[106624]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:32.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:32 smithi099 ceph-mon[94036]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:32.799 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:32.799 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:33.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:33 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3453909251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:33.548 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:33 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3453909251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:33.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:33 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3453909251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:34.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:34 smithi029 ceph-mon[102350]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:34.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:34.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:34 smithi029 ceph-mon[106624]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:34.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:34.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:34 smithi099 ceph-mon[94036]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:34.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:35.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:55:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:55:34] "GET /metrics HTTP/1.1" 200 34087 "" "Prometheus/2.43.0" 2024-03-29T04:55:36.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:36 smithi029 ceph-mon[102350]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:36.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:36 smithi029 ceph-mon[106624]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:36.549 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:36.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:36 smithi099 ceph-mon[94036]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:36.826 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:38.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:38 smithi029 ceph-mon[106624]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:38.350 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:38 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:38.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:38 smithi029 ceph-mon[102350]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:38.350 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:38 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:38.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:38 smithi099 ceph-mon[94036]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:38.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:38 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:38.873 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:38.873 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:39.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:39 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/481940083' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:39.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:39 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/481940083' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:39.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:39 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/481940083' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:40.448 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:40 smithi029 ceph-mon[102350]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:40.448 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:40 smithi029 ceph-mon[106624]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:40.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:40 smithi099 ceph-mon[94036]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:42.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:42 smithi029 ceph-mon[106624]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:42.433 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:42 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:42.434 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:42 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:42.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:42 smithi029 ceph-mon[102350]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:42.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:42.434 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:42 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:42.573 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:42.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:42 smithi099 ceph-mon[94036]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:42.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:42 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:42.628 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:42 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:42.866 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[102350]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.082 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[106624]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:44.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:44.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:44.083 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:43 smithi099 ceph-mon[94036]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:44.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:44.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:44.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:55:44.890 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:44.890 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:45.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:44 smithi099 ceph-mon[94036]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:45.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:44 smithi099 ceph-mon[94036]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:45.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:44 smithi029 ceph-mon[102350]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:45.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:44 smithi029 ceph-mon[102350]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:45.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:44 smithi029 ceph-mon[106624]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:45.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:44 smithi029 ceph-mon[106624]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:45.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:55:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:55:44] "GET /metrics HTTP/1.1" 200 34087 "" "Prometheus/2.43.0" 2024-03-29T04:55:45.949 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:45 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4148389933' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:45.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:45 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4148389933' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:46.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:45 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4148389933' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:47.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:46 smithi099 ceph-mon[94036]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:47.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:46 smithi029 ceph-mon[102350]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:47.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:46 smithi029 ceph-mon[106624]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:48.617 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:48.895 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:48.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:48 smithi029 ceph-mon[102350]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:48.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:48 smithi029 ceph-mon[106624]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:49.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:48 smithi099 ceph-mon[94036]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:50.041 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:50.041 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:50.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:50.880 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:50.881 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:50.881 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:50 smithi029 ceph-mon[102350]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:50.882 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:50 smithi029 ceph-mon[106624]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:51.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:50 smithi099 ceph-mon[94036]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:52.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:51 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2476430317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:52.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:51 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2476430317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:52.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:51 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2476430317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:53.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:52 smithi099 ceph-mon[94036]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:53.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:52 smithi029 ceph-mon[106624]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:53.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:52 smithi029 ceph-mon[102350]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:54.495 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:54.769 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:55.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:54 smithi099 ceph-mon[94036]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:55.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:54 smithi029 ceph-mon[106624]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:55.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:54 smithi029 ceph-mon[102350]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:55.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:55:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:55:54] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2024-03-29T04:55:56.941 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:55:56.941 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:56.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:56 smithi029 ceph-mon[102350]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:56.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:56 smithi029 ceph-mon[106624]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:57.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:56 smithi099 ceph-mon[94036]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:57.941 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:57 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3684131748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:57.941 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:57 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3684131748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:58.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:57 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3684131748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:59.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:55:58 smithi099 ceph-mon[94036]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:59.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:55:58 smithi029 ceph-mon[102350]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:55:59.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:55:58 smithi029 ceph-mon[106624]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:00.661 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:00.941 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:00.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:00 smithi029 ceph-mon[102350]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:00.948 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:00 smithi029 ceph-mon[106624]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:01.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:00 smithi099 ceph-mon[94036]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:02.939 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:02 smithi029 ceph-mon[102350]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:02.939 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:02 smithi029 ceph-mon[106624]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:03.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:02 smithi099 ceph-mon[94036]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:03.308 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:03.308 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:03.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:03 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2485147380' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:03.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:03 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2485147380' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:04.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:03 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2485147380' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:05.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:04 smithi099 ceph-mon[94036]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:05.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:05.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:56:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:56:04] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2024-03-29T04:56:05.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:04 smithi029 ceph-mon[102350]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:05.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:05.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:04 smithi029 ceph-mon[106624]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:05.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:06.993 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:07.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:06 smithi099 ceph-mon[94036]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:07.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:06 smithi029 ceph-mon[102350]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:07.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:06 smithi029 ceph-mon[106624]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:07.268 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:08.929 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:08 smithi029 ceph-mon[102350]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:08.930 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:08 smithi029 ceph-mon[106624]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:09.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:08 smithi099 ceph-mon[94036]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:09.455 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:09.456 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:10.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2437239567' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:10.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2437239567' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:10.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2437239567' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:11.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:10 smithi099 ceph-mon[94036]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:11.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:10 smithi029 ceph-mon[102350]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:11.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:10 smithi029 ceph-mon[106624]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:13.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:12 smithi099 ceph-mon[94036]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:13.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:12 smithi029 ceph-mon[102350]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:13.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:12 smithi029 ceph-mon[106624]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:13.279 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:13.562 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:15.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:14 smithi099 ceph-mon[94036]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:15.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:14 smithi029 ceph-mon[102350]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:15.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:14 smithi029 ceph-mon[106624]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:15.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:56:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:56:14] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2024-03-29T04:56:15.743 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:15.744 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:16.113 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3422524076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:16.113 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3422524076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:16.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3422524076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:17.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:16 smithi099 ceph-mon[94036]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:17.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:16 smithi029 ceph-mon[102350]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:17.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:16 smithi029 ceph-mon[106624]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:19.128 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:18 smithi099 ceph-mon[94036]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:19.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:18 smithi029 ceph-mon[102350]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:19.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:18 smithi029 ceph-mon[106624]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:19.570 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:19.855 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:20.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:20.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:20.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:21.184 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:20 smithi029 ceph-mon[102350]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:21.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:20 smithi029 ceph-mon[106624]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:21.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:20 smithi099 ceph-mon[94036]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:21.978 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:21.978 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:23.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:22 smithi029 ceph-mon[106624]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:23.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:22 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3814314693' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:23.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:22 smithi029 ceph-mon[102350]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:23.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:22 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3814314693' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:23.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:22 smithi099 ceph-mon[94036]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:23.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:22 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3814314693' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:25.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:24 smithi029 ceph-mon[102350]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:25.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:24 smithi029 ceph-mon[106624]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:25.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:56:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:56:24] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.43.0" 2024-03-29T04:56:25.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:24 smithi099 ceph-mon[94036]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:25.758 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:26.032 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:27.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:26 smithi029 ceph-mon[102350]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:27.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:26 smithi029 ceph-mon[106624]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:27.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:26 smithi099 ceph-mon[94036]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:28.181 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:28.181 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:29.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:28 smithi029 ceph-mon[102350]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:29.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:28 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/678961989' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:29.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:28 smithi029 ceph-mon[106624]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:29.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:28 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/678961989' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:29.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:28 smithi099 ceph-mon[94036]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:29.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:28 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/678961989' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:31.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:30 smithi029 ceph-mon[102350]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:31.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:30 smithi029 ceph-mon[106624]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:31.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:30 smithi099 ceph-mon[94036]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:32.012 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:32.287 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:33.188 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:32 smithi029 ceph-mon[102350]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:33.188 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:32 smithi029 ceph-mon[106624]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:33.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:32 smithi099 ceph-mon[94036]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:34.464 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:34.464 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:35.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-mon[102350]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:35.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:35.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1820076675' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:35.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:56:34] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:56:35.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-mon[106624]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:35.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:35.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:34 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1820076675' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:35.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:34 smithi099 ceph-mon[94036]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:35.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:35.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:34 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1820076675' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:37.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:36 smithi029 ceph-mon[102350]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:37.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:36 smithi029 ceph-mon[106624]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:37.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:36 smithi099 ceph-mon[94036]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:38.233 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:38.506 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:39.181 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:38 smithi099 ceph-mon[94036]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:39.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:38 smithi029 ceph-mon[102350]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:39.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:38 smithi029 ceph-mon[106624]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:40.554 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:40.555 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:41.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:40 smithi029 ceph-mon[106624]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:41.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:40 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/628025593' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:41.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:40 smithi029 ceph-mon[102350]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:41.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:40 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/628025593' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:41.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:40 smithi099 ceph-mon[94036]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:41.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:40 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/628025593' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:43.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:42 smithi029 ceph-mon[102350]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:43.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:42 smithi029 ceph-mon[106624]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:43.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:42 smithi099 ceph-mon[94036]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:44.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:43 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:44.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:43 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:44.368 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:44.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:43 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:44.948 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:44 smithi029 ceph-mon[102350]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:44.949 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:44 smithi029 ceph-mon[106624]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:44.949 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:56:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:56:44] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:56:45.129 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:44 smithi099 ceph-mon[94036]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:45.138 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:47.080 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:47.080 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:47.093 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:46 smithi029 ceph-mon[102350]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:47.093 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:46 smithi029 ceph-mon[106624]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:47.287 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:46 smithi099 ceph-mon[94036]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:48.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:47 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2278691018' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:48.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:47 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:48.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:47 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:48.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:47 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2278691018' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:48.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:47 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:48.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:47 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:48.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:47 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2278691018' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:48.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:47 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:48.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:47 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:49.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:48 smithi029 ceph-mon[102350]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:49.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:48 smithi029 ceph-mon[106624]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:49.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:48 smithi099 ceph-mon[94036]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:50.198 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:50.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:50.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:50.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:50.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:50.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:50.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:50.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:50.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:50.898 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:51.161 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[102350]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[106624]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:51.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:50 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:51.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:50 smithi099 ceph-mon[94036]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:51.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:51.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:51.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:50 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:56:52.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:51 smithi029 ceph-mon[102350]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:52.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:51 smithi029 ceph-mon[102350]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:52.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:51 smithi029 ceph-mon[106624]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:52.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:51 smithi029 ceph-mon[106624]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:52.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:51 smithi099 ceph-mon[94036]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:52.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:51 smithi099 ceph-mon[94036]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:53.371 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:53.371 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:54.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:53 smithi029 ceph-mon[102350]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:54.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:53 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/61094060' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:54.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:53 smithi029 ceph-mon[106624]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:54.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:53 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/61094060' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:54.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:53 smithi099 ceph-mon[94036]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:54.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:53 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/61094060' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:55.198 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:56:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:56:54] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:56:56.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:55 smithi029 ceph-mon[102350]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:56.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:55 smithi029 ceph-mon[106624]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:56.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:55 smithi099 ceph-mon[94036]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:57.154 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:57.420 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:58.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:57 smithi029 ceph-mon[102350]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:58.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:57 smithi029 ceph-mon[106624]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:58.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:57 smithi099 ceph-mon[94036]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:56:59.573 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:56:59.573 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:00.198 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:59 smithi029 ceph-mon[106624]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:00.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:56:59 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2016669773' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:00.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:59 smithi029 ceph-mon[102350]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:00.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:56:59 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2016669773' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:00.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:59 smithi099 ceph-mon[94036]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:00.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:56:59 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2016669773' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:02.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:01 smithi029 ceph-mon[102350]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:02.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:01 smithi029 ceph-mon[106624]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:02.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:01 smithi099 ceph-mon[94036]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:03.447 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:03.724 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:04.174 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:03 smithi029 ceph-mon[102350]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:04.174 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:03 smithi029 ceph-mon[106624]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:04.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:03 smithi099 ceph-mon[94036]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:05.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:05.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:05.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:57:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:57:04] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2024-03-29T04:57:05.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:05.751 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:05.751 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:06.136 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:05 smithi029 ceph-mon[102350]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:06.137 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:05 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/110038342' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:06.137 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:05 smithi029 ceph-mon[106624]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:06.137 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:05 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/110038342' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:06.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:05 smithi099 ceph-mon[94036]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:06.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:05 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/110038342' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:08.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:07 smithi099 ceph-mon[94036]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:08.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:07 smithi029 ceph-mon[102350]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:08.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:07 smithi029 ceph-mon[106624]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:09.598 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:09.874 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:10.302 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:09 smithi029 ceph-mon[102350]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:10.303 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:09 smithi029 ceph-mon[106624]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:10.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:09 smithi099 ceph-mon[94036]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:12.234 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:12.235 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:12.235 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:11 smithi029 ceph-mon[102350]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:12.235 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:11 smithi029 ceph-mon[106624]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:12.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:11 smithi099 ceph-mon[94036]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:13.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:12 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/176964251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:13.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:12 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/176964251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:13.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:12 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/176964251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:14.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:13 smithi099 ceph-mon[94036]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:14.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:13 smithi029 ceph-mon[102350]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:14.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:13 smithi029 ceph-mon[106624]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:15.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:57:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:57:14] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.43.0" 2024-03-29T04:57:16.092 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:16.367 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:16.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:15 smithi099 ceph-mon[94036]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:16.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:15 smithi029 ceph-mon[102350]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:16.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:15 smithi029 ceph-mon[106624]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:18.283 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:17 smithi029 ceph-mon[102350]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:18.283 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:17 smithi029 ceph-mon[106624]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:18.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:17 smithi099 ceph-mon[94036]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:18.527 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:18.527 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:19.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:18 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3545718159' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:19.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:18 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3545718159' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:19.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:18 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3545718159' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:20.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:19 smithi099 ceph-mon[94036]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:20.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:20.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:19 smithi029 ceph-mon[102350]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:20.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:20.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:19 smithi029 ceph-mon[106624]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:20.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:22.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:21 smithi099 ceph-mon[94036]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:22.382 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:22.412 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:21 smithi029 ceph-mon[102350]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:22.412 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:21 smithi029 ceph-mon[106624]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:22.656 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:24.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:23 smithi029 ceph-mon[102350]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:24.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:23 smithi029 ceph-mon[106624]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:24.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:23 smithi099 ceph-mon[94036]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:24.782 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:24.783 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:25.175 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:24 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/908620898' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:25.176 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:57:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:57:24] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:57:25.176 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:24 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/908620898' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:25.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:24 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/908620898' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:26.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:25 smithi099 ceph-mon[94036]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:26.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:25 smithi029 ceph-mon[102350]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:26.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:25 smithi029 ceph-mon[106624]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:28.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:27 smithi099 ceph-mon[94036]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:28.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:27 smithi029 ceph-mon[106624]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:28.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:27 smithi029 ceph-mon[102350]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:28.672 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:28.936 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:30.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:29 smithi029 ceph-mon[102350]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:30.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:29 smithi029 ceph-mon[106624]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:30.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:29 smithi099 ceph-mon[94036]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:31.137 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:31.137 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:32.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:31 smithi029 ceph-mon[102350]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:32.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:31 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/1909844747' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:32.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:31 smithi029 ceph-mon[106624]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:32.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:31 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/1909844747' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:32.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:31 smithi099 ceph-mon[94036]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:32.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:31 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/1909844747' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:34.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:34 smithi099 ceph-mon[94036]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:34.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:34 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:34.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:34 smithi029 ceph-mon[102350]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:34.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:34 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:34.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:34 smithi029 ceph-mon[106624]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:34.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:34 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:34.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:35.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:57:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:57:34] "GET /metrics HTTP/1.1" 200 34087 "" "Prometheus/2.43.0" 2024-03-29T04:57:35.208 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:36.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:36 smithi029 ceph-mon[102350]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:36.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:36 smithi029 ceph-mon[106624]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:36.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:36 smithi099 ceph-mon[94036]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:37.519 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:38.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:38 smithi029 ceph-mon[102350]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:38.168 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:38 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2625145390' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:38.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:38 smithi029 ceph-mon[106624]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:38.168 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:38 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2625145390' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:38.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:38 smithi099 ceph-mon[94036]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:38.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:38 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2625145390' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:40.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:40 smithi099 ceph-mon[94036]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:40.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:40 smithi029 ceph-mon[102350]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:40.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:40 smithi029 ceph-mon[106624]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:41.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:41.649 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:42.073 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:42 smithi029 ceph-mon[102350]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:42.074 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:42 smithi029 ceph-mon[106624]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:42.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:42 smithi099 ceph-mon[94036]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:44.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:44 smithi099 ceph-mon[94036]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:44.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:44 smithi029 ceph-mon[102350]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:44.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:44 smithi029 ceph-mon[106624]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:45.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:57:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:57:44] "GET /metrics HTTP/1.1" 200 34087 "" "Prometheus/2.43.0" 2024-03-29T04:57:46.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:46 smithi099 ceph-mon[94036]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:46.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:46 smithi029 ceph-mon[102350]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:46.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:46 smithi029 ceph-mon[106624]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:48.090 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:48.090 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:48.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:48 smithi099 ceph-mon[94036]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:48.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:48 smithi029 ceph-mon[102350]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:48.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:48 smithi029 ceph-mon[106624]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:49.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:49 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3403616117' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:49.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:49 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:49.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:49 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3403616117' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:49.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:49 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:49.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:49 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3403616117' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:49.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:49 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:50.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:50 smithi099 ceph-mon[94036]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:50.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:50 smithi029 ceph-mon[102350]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:50.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:50 smithi029 ceph-mon[106624]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:51.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:51 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:51.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:51 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:51.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:51 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:51.969 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:52.337 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[106624]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[102350]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.338 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:52 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:52 smithi099 ceph-mon[94036]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:52.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:52 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:52.690 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:54.373 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:54 smithi029 ceph-mon[102350]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:54.373 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:54 smithi029 ceph-mon[106624]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:54.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:54 smithi099 ceph-mon[94036]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:54.938 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:57:54.939 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:54.939 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:57:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:57:54] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.43.0" 2024-03-29T04:57:55.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:55 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/4004993472' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:55.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:55 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/4004993472' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:55.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:55 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/4004993472' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:56.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:56 smithi099 ceph-mon[94036]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:56.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:56 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:56.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:56 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:56.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:56 smithi029 ceph-mon[106624]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:56.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:56 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:56.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:56 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:56.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:56 smithi029 ceph-mon[102350]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:56.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:56 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:56.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:56 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:58.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:58 smithi099 ceph-mon[94036]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:58.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:58 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:58.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:58 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:58.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:58 smithi029 ceph-mon[102350]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:58.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:58.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:58 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:58.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:58 smithi029 ceph-mon[106624]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:57:58.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:58 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:58.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:58 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:59.326 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:59.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:59 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:59.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:59 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:59.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:57:59 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:59.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:59 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:59.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:59 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:59.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:57:59 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:59.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:59 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:59.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:59 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:59.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:57:59 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' 2024-03-29T04:57:59.601 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:58:00.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:00 smithi099 ceph-mon[94036]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:00 smithi099 ceph-mon[94036]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:00 smithi099 ceph-mon[94036]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:00 smithi029 ceph-mon[102350]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:00 smithi029 ceph-mon[102350]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:00 smithi029 ceph-mon[102350]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:00 smithi029 ceph-mon[106624]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:00 smithi029 ceph-mon[106624]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:00.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:00 smithi029 ceph-mon[106624]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:01.731 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:58:01.731 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:58:02.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:02 smithi099 ceph-mon[94036]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:02.380 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:02 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3443499743' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:02.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:02 smithi029 ceph-mon[106624]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:02.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:02 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3443499743' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:02.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:02 smithi029 ceph-mon[102350]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:02.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:02 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3443499743' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:04.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:04 smithi029 ceph-mon[106624]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:04.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:04 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:04.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:04 smithi029 ceph-mon[102350]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:04.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:04 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:04.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:04 smithi099 ceph-mon[94036]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:04.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:04 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:05.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:58:04] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:58:05.553 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:58:05.826 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:58:06.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:06 smithi029 ceph-mon[102350]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:06.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:06 smithi029 ceph-mon[106624]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:06.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:06 smithi099 ceph-mon[94036]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:08.165 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:58:08.165 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:58:08.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:08 smithi029 ceph-mon[102350]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:08.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:08 smithi029 ceph-mon[106624]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:08.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:08 smithi099 ceph-mon[94036]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:09.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:09 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/2572542763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:09.199 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:09 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/2572542763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:09.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:09 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/2572542763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:10.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:10 smithi029 ceph-mon[102350]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:10.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:10 smithi029 ceph-mon[106624]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:10.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:10 smithi099 ceph-mon[94036]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:11.979 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:58:12.257 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:58:12.379 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:12 smithi029 ceph-mon[102350]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:12.379 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:12 smithi029 ceph-mon[106624]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:12.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:12 smithi099 ceph-mon[94036]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:14.382 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:14 smithi029 ceph-mon[102350]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:14.382 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:14 smithi029 ceph-mon[106624]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:14.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:14 smithi099 ceph-mon[94036]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:14.640 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:58:14.640 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:58:15.199 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:15 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/861650436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:15.200 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:15 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/861650436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:15.200 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:58:14] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:58:15.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:15 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/861650436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:16.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:16 smithi029 ceph-mon[102350]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:16.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:16 smithi029 ceph-mon[106624]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:16.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:16 smithi099 ceph-mon[94036]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:18.422 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:58:18.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:18 smithi029 ceph-mon[102350]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:18.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:18 smithi029 ceph-mon[106624]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:18.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:18 smithi099 ceph-mon[94036]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:18.697 INFO:teuthology.orchestra.run.smithi029.stderr:Inferring config /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:58:19.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:19 smithi029 ceph-mon[102350]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:19.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:19 smithi029 ceph-mon[106624]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:19.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:19 smithi099 ceph-mon[94036]: from='mgr.25229 172.21.15.29:0/3421487698' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:20.314 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:20 smithi029 ceph-mon[102350]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:20.314 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:20 smithi029 ceph-mon[106624]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:20.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:20 smithi099 ceph-mon[94036]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:20.843 INFO:teuthology.orchestra.run.smithi029.stdout: 2024-03-29T04:58:20.844 INFO:teuthology.orchestra.run.smithi029.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:58:21.449 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:21 smithi029 ceph-mon[102350]: from='client.? 172.21.15.29:0/3188745181' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:21.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:21 smithi029 ceph-mon[106624]: from='client.? 172.21.15.29:0/3188745181' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:21.543 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-29T04:58:21.774 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=4905ab564e4b4074bd94e9b52c1da1df Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-29T04:58:21.776 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-29T04:58:21.786 INFO:tasks.cephadm:Teardown begin 2024-03-29T04:58:21.786 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:58:21.790 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:21 smithi099 ceph-mon[94036]: from='client.? 172.21.15.29:0/3188745181' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:58:21.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:58:21.865 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-29T04:58:21.865 DEBUG:teuthology.orchestra.run.smithi029:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:58:21.905 DEBUG:teuthology.orchestra.run.smithi099:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:58:21.920 INFO:tasks.cephadm:Stopping all daemons... 2024-03-29T04:58:21.920 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-03-29T04:58:21.921 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a 2024-03-29T04:58:22.166 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:21 smithi029 systemd[1]: Stopping Ceph mon.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:22.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-mon[106624]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:22.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-mon[102350]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:22.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[102324]: 2024-03-29T04:58:22.266+0000 7f9a3200a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:22.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a[102324]: 2024-03-29T04:58:22.266+0000 7f9a3200a700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-03-29T04:58:22.450 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:22 smithi029 podman[162741]: 2024-03-29 04:58:22.308360612 +0000 UTC m=+0.175167674 container died 4ef356385530cdfeaab681d8cf4dbf04a593410c6487f600debd63057c46fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-29T04:58:22.450 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:58:22] ENGINE Bus STOPPING 2024-03-29T04:58:22.451 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:58:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:58:22.451 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:58:22] ENGINE Bus STOPPED 2024-03-29T04:58:22.451 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:58:22] ENGINE Bus STARTING 2024-03-29T04:58:22.451 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:58:22] ENGINE Serving on http://:::9283 2024-03-29T04:58:22.451 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:22 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: [29/Mar/2024:04:58:22] ENGINE Bus STARTED 2024-03-29T04:58:22.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:22 smithi099 ceph-mon[94036]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:22.949 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:22 smithi029 podman[162741]: 2024-03-29 04:58:22.758687723 +0000 UTC m=+0.625494776 container cleanup 4ef356385530cdfeaab681d8cf4dbf04a593410c6487f600debd63057c46fc96 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-03-29T04:58:22.949 INFO:journalctl@ceph.mon.a.smithi029.stdout:Mar 29 04:58:22 smithi029 bash[162741]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:58:23.240 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.a.service' 2024-03-29T04:58:23.292 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:23.292 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-03-29T04:58:23.292 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-03-29T04:58:23.293 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.c 2024-03-29T04:58:23.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:23 smithi029 systemd[1]: Stopping Ceph mon.c for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:23.856 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c[106601]: 2024-03-29T04:58:23.592+0000 7f32ae81f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:23.857 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:23 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c[106601]: 2024-03-29T04:58:23.592+0000 7f32ae81f700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-03-29T04:58:24.184 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:23 smithi029 podman[162882]: 2024-03-29 04:58:23.850210059 +0000 UTC m=+0.380793077 container died 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:58:24.449 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:24 smithi029 podman[162882]: 2024-03-29 04:58:24.361381022 +0000 UTC m=+0.891964083 container cleanup 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-29T04:58:24.450 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:24 smithi029 bash[162882]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c 2024-03-29T04:58:24.912 INFO:journalctl@ceph.mon.c.smithi029.stdout:Mar 29 04:58:24 smithi029 podman[162901]: 2024-03-29 04:58:24.649493795 +0000 UTC m=+0.797268010 container remove 69712aa77faef5d8d868c03d0f4d415346436e529d35573e468e9cba7f61d54e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-c, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-29T04:58:24.956 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.c.service' 2024-03-29T04:58:24.992 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:24.992 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-03-29T04:58:24.993 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-03-29T04:58:24.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.b 2024-03-29T04:58:25.199 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: ::ffff:172.21.15.99 - - [29/Mar/2024:04:58:24] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2024-03-29T04:58:25.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:25 smithi099 systemd[1]: Stopping Ceph mon.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:25.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b[94013]: 2024-03-29T04:58:25.308+0000 7f9ff6d71700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:25.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b[94013]: 2024-03-29T04:58:25.308+0000 7f9ff6d71700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-03-29T04:58:25.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:25 smithi099 podman[106730]: 2024-03-29 04:58:25.493525698 +0000 UTC m=+0.314650991 container died a53b591d5ed70b95a39e768892e628f788199907f5a44105e74403049ad7595a (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS) 2024-03-29T04:58:26.130 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:25 smithi099 podman[106730]: 2024-03-29 04:58:25.949040821 +0000 UTC m=+0.770166112 container cleanup a53b591d5ed70b95a39e768892e628f788199907f5a44105e74403049ad7595a (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-29T04:58:26.130 INFO:journalctl@ceph.mon.b.smithi099.stdout:Mar 29 04:58:25 smithi099 bash[106730]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mon-b 2024-03-29T04:58:26.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mon.b.service' 2024-03-29T04:58:26.488 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:26.488 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-03-29T04:58:26.489 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-03-29T04:58:26.489 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y 2024-03-29T04:58:26.812 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:26 smithi029 systemd[1]: Stopping Ceph mgr.y for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:27.105 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:58:26.804+0000 7f94ee275700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:27.105 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:26 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y[92755]: 2024-03-29T04:58:26.804+0000 7f94ee275700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-03-29T04:58:27.106 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:26 smithi029 podman[163012]: 2024-03-29 04:58:26.836458162 +0000 UTC m=+0.163067423 container died 280a22bdfeadcd1cb633d24990ee4b29f51a353343d91d35892254dd54d2b0d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-29T04:58:27.699 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:27 smithi029 podman[163012]: 2024-03-29 04:58:27.383001779 +0000 UTC m=+0.709611034 container cleanup 280a22bdfeadcd1cb633d24990ee4b29f51a353343d91d35892254dd54d2b0d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-29T04:58:27.700 INFO:journalctl@ceph.mgr.y.smithi029.stdout:Mar 29 04:58:27 smithi029 bash[163012]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-y 2024-03-29T04:58:27.892 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.y.service' 2024-03-29T04:58:27.931 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:27.932 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-03-29T04:58:27.932 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-03-29T04:58:27.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x 2024-03-29T04:58:28.269 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:58:27 smithi099 systemd[1]: Stopping Ceph mgr.x for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:28.528 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:58:28 smithi099 podman[106862]: 2024-03-29 04:58:28.262760943 +0000 UTC m=+0.145538944 container died 1c6cc4c8d8c3338929b33f1155bc548ad14511fcb50025d663b581928e23c056 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:58:28.879 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:58:28 smithi099 podman[106862]: 2024-03-29 04:58:28.620596956 +0000 UTC m=+0.503374952 container cleanup 1c6cc4c8d8c3338929b33f1155bc548ad14511fcb50025d663b581928e23c056 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:58:28.880 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Mar 29 04:58:28 smithi099 bash[106862]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:58:29.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@mgr.x.service' 2024-03-29T04:58:29.082 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:29.082 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-03-29T04:58:29.082 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-29T04:58:29.083 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.0 2024-03-29T04:58:29.417 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:29 smithi029 systemd[1]: Stopping Ceph osd.0 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:29.699 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[116410]: 2024-03-29T04:58:29.409+0000 7f0dbb48e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:29.700 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[116410]: 2024-03-29T04:58:29.409+0000 7f0dbb48e700 -1 osd.0 131 *** Got signal Terminated *** 2024-03-29T04:58:29.700 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:29 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0[116410]: 2024-03-29T04:58:29.409+0000 7f0dbb48e700 -1 osd.0 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:35.327 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:34 smithi029 podman[163157]: 2024-03-29 04:58:34.996285324 +0000 UTC m=+5.727247574 container died 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-29T04:58:35.699 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:35 smithi029 podman[163157]: 2024-03-29 04:58:35.520457672 +0000 UTC m=+6.251419929 container cleanup 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-8da2362) 2024-03-29T04:58:35.700 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:35 smithi029 bash[163157]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0 2024-03-29T04:58:36.129 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:35 smithi029 podman[163179]: 2024-03-29 04:58:35.791636182 +0000 UTC m=+0.794161448 container remove 8456d3dfc6078e6fe1129981fa5faf0da4f79747b2776ce2399fc1c8968d1cb9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-29T04:58:36.404 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:36 smithi029 podman[163275]: 2024-03-29 04:58:36.025574375 +0000 UTC m=+0.017853699 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:58:36.405 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:36 smithi029 podman[163275]: 2024-03-29 04:58:36.151750646 +0000 UTC m=+0.144029953 container create 870d8bf140bd9222394b557c130d5d60acfeae595a370aad4ad6a8d8b2f0bd22 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True) 2024-03-29T04:58:36.699 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:36 smithi029 podman[163275]: 2024-03-29 04:58:36.398061528 +0000 UTC m=+0.390340851 container init 870d8bf140bd9222394b557c130d5d60acfeae595a370aad4ad6a8d8b2f0bd22 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-29T04:58:36.700 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:36 smithi029 podman[163275]: 2024-03-29 04:58:36.40399266 +0000 UTC m=+0.396271971 container start 870d8bf140bd9222394b557c130d5d60acfeae595a370aad4ad6a8d8b2f0bd22 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-8da2362) 2024-03-29T04:58:36.700 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:36 smithi029 podman[163275]: 2024-03-29 04:58:36.497817993 +0000 UTC m=+0.490097310 container attach 870d8bf140bd9222394b557c130d5d60acfeae595a370aad4ad6a8d8b2f0bd22 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-29T04:58:36.973 INFO:journalctl@ceph.osd.0.smithi029.stdout:Mar 29 04:58:36 smithi029 podman[163275]: 2024-03-29 04:58:36.699471786 +0000 UTC m=+0.691751095 container died 870d8bf140bd9222394b557c130d5d60acfeae595a370aad4ad6a8d8b2f0bd22 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-0-deactivate, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, io.buildah.version=1.33.5) 2024-03-29T04:58:37.529 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.0.service' 2024-03-29T04:58:37.575 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:37.575 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-29T04:58:37.575 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-29T04:58:37.576 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.1 2024-03-29T04:58:37.870 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:37 smithi029 systemd[1]: Stopping Ceph osd.1 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:38.171 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:37 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[124752]: 2024-03-29T04:58:37.863+0000 7f67c024e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:38.172 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:37 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[124752]: 2024-03-29T04:58:37.863+0000 7f67c024e700 -1 osd.1 131 *** Got signal Terminated *** 2024-03-29T04:58:38.172 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:37 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1[124752]: 2024-03-29T04:58:37.863+0000 7f67c024e700 -1 osd.1 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:40.449 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:40 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:40.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:58:40.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:40 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:40.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:58:43.621 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:43 smithi029 podman[163408]: 2024-03-29 04:58:43.269332903 +0000 UTC m=+5.511778724 container died cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-29T04:58:43.950 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:43 smithi029 podman[163408]: 2024-03-29 04:58:43.782037874 +0000 UTC m=+6.024483684 container cleanup cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:58:43.950 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:43 smithi029 bash[163408]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1 2024-03-29T04:58:44.418 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:44 smithi029 podman[163431]: 2024-03-29 04:58:44.10357833 +0000 UTC m=+0.819423877 container remove cd71767b2146b906cb3ef70fb1def0bc8069415176cbf1a0dc304f88634c8774 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD) 2024-03-29T04:58:44.418 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:44.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:58:44.418 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:44 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:44.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:58:44.699 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:44 smithi029 podman[163528]: 2024-03-29 04:58:44.314100171 +0000 UTC m=+0.018000129 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:58:44.700 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:44 smithi029 podman[163528]: 2024-03-29 04:58:44.43030617 +0000 UTC m=+0.134206157 container create f276d828128d48b5ac8722b5872a2d89009c8fc7fff041200073893b6b1dff0f (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, ceph=True) 2024-03-29T04:58:45.191 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:44 smithi029 podman[163528]: 2024-03-29 04:58:44.884792169 +0000 UTC m=+0.588692130 container init f276d828128d48b5ac8722b5872a2d89009c8fc7fff041200073893b6b1dff0f (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-29T04:58:45.191 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:44 smithi029 podman[163528]: 2024-03-29 04:58:44.890154872 +0000 UTC m=+0.594054817 container start f276d828128d48b5ac8722b5872a2d89009c8fc7fff041200073893b6b1dff0f (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-29T04:58:45.191 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:44 smithi029 podman[163528]: 2024-03-29 04:58:44.984911329 +0000 UTC m=+0.688811291 container attach f276d828128d48b5ac8722b5872a2d89009c8fc7fff041200073893b6b1dff0f (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, ceph=True, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T04:58:45.449 INFO:journalctl@ceph.osd.1.smithi029.stdout:Mar 29 04:58:45 smithi029 podman[163528]: 2024-03-29 04:58:45.18458547 +0000 UTC m=+0.888485424 container died f276d828128d48b5ac8722b5872a2d89009c8fc7fff041200073893b6b1dff0f (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_CLEAN=True) 2024-03-29T04:58:45.844 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.1.service' 2024-03-29T04:58:45.886 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:45.886 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-29T04:58:45.886 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-29T04:58:45.886 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.2 2024-03-29T04:58:46.200 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:45 smithi029 systemd[1]: Stopping Ceph osd.2 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:46.699 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[133795]: 2024-03-29T04:58:46.249+0000 7f2bddba1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:46.699 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[133795]: 2024-03-29T04:58:46.249+0000 7f2bddba1700 -1 osd.2 131 *** Got signal Terminated *** 2024-03-29T04:58:46.700 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:46 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2[133795]: 2024-03-29T04:58:46.249+0000 7f2bddba1700 -1 osd.2 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:50.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:50 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:50.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:58:50.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:50 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:50.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:58:50.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:50 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:50.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:58:51.925 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:51 smithi029 podman[163659]: 2024-03-29 04:58:51.598527702 +0000 UTC m=+5.498609441 container died 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:58:52.200 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 podman[163659]: 2024-03-29 04:58:52.109463113 +0000 UTC m=+6.009544858 container cleanup 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362) 2024-03-29T04:58:52.200 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 bash[163659]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2 2024-03-29T04:58:52.699 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 podman[163683]: 2024-03-29 04:58:52.3708471 +0000 UTC m=+0.769372565 container remove 7299367ca14b97fb2f7068afd043262be835b524c33c7991e5a16f89eeb8fffd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-29T04:58:52.994 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 podman[163778]: 2024-03-29 04:58:52.628497758 +0000 UTC m=+0.017607511 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:58:52.994 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 podman[163778]: 2024-03-29 04:58:52.749138595 +0000 UTC m=+0.138248330 container create ae46eb651411d6851da22e9f14f54da0a0af7f36deadf2b6b0bf5f3f5043acae (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-29T04:58:52.994 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 podman[163778]: 2024-03-29 04:58:52.98692755 +0000 UTC m=+0.376037276 container init ae46eb651411d6851da22e9f14f54da0a0af7f36deadf2b6b0bf5f3f5043acae (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362) 2024-03-29T04:58:53.293 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:52 smithi029 podman[163778]: 2024-03-29 04:58:52.992173249 +0000 UTC m=+0.381282976 container start ae46eb651411d6851da22e9f14f54da0a0af7f36deadf2b6b0bf5f3f5043acae (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:53.293 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:53 smithi029 podman[163778]: 2024-03-29 04:58:53.11227928 +0000 UTC m=+0.501389018 container attach ae46eb651411d6851da22e9f14f54da0a0af7f36deadf2b6b0bf5f3f5043acae (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-29T04:58:53.607 INFO:journalctl@ceph.osd.2.smithi029.stdout:Mar 29 04:58:53 smithi029 podman[163778]: 2024-03-29 04:58:53.287230414 +0000 UTC m=+0.676340154 container died ae46eb651411d6851da22e9f14f54da0a0af7f36deadf2b6b0bf5f3f5043acae (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:58:53.930 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.2.service' 2024-03-29T04:58:53.974 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:53.975 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-29T04:58:53.975 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-29T04:58:53.975 DEBUG:teuthology.orchestra.run.smithi029:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.3 2024-03-29T04:58:54.199 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:58:54 smithi029 systemd[1]: Stopping Ceph osd.3 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:58:54.700 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:58:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:58:54.317+0000 7f57251df700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:54.700 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:58:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:58:54.317+0000 7f57251df700 -1 osd.3 131 *** Got signal Terminated *** 2024-03-29T04:58:54.700 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:58:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:58:54.317+0000 7f57251df700 -1 osd.3 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:54.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:54.316Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:58:54.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:54.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:58:54.701 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:58:54 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:58:54.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:58:56.630 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:58:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:58:56.122Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-03-29T04:58:56.630 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:58:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:58:56.122Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-03-29T04:58:56.630 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:58:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:58:56.122Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-03-29T04:58:56.631 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:58:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:58:56.133Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.29:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.29:8765: connect: connection refused" 2024-03-29T04:58:57.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:58:56 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:58:56.681+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:58:57.581 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:58:57 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:58:57.347+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:58:57.861 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:58:57 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:58:57.710+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:58:58.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:58:57 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:58:57.854+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:58:58.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:58:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:58:58.314+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:58:59.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:58:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:58:58.860+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:58:59.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:58:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:58:58.727+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:58:59.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:58:58 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:58:58.845+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:58:59.449 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:58:59 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3[139351]: 2024-03-29T04:58:59.113+0000 7f57217d3700 -1 osd.3 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.307190+0000 front 2024-03-29T04:58:32.307243+0000 (oldest deadline 2024-03-29T04:58:58.207255+0000) 2024-03-29T04:58:59.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:58:59 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:58:59.349+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:00.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:58:59 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:58:59.756+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:00.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:58:59 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:58:59.860+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:00.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:58:59 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:58:59.854+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:00.167 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:58:59 smithi029 podman[163909]: 2024-03-29 04:58:59.853248972 +0000 UTC m=+5.658814757 container died e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, ceph=True) 2024-03-29T04:59:00.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:00.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:00.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:00.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:00.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:00 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:00.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:00.791 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:00 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:00.398+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:00.828 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:00 smithi029 podman[163909]: 2024-03-29 04:59:00.560973819 +0000 UTC m=+6.366539596 container cleanup e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5) 2024-03-29T04:59:00.829 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:00 smithi029 bash[163909]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3 2024-03-29T04:59:01.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:00 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:00.895+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:01.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:00 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:00.783+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:01.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:00 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:00.893+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:01.196 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:00 smithi029 podman[163930]: 2024-03-29 04:59:00.821643524 +0000 UTC m=+0.963037636 container remove e0e58b4f3af9ce18041b443f5a912009e3df1689f7e54a0aef265b339cbf7f71 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-29T04:59:01.450 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:01 smithi029 podman[164036]: 2024-03-29 04:59:01.092586016 +0000 UTC m=+0.018136130 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:01.450 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:01 smithi029 podman[164036]: 2024-03-29 04:59:01.198189227 +0000 UTC m=+0.123739313 container create ecfec7053880cfe4db32b22b78c5a02daaff500d55af5b9fb40d67e621350883 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-29T04:59:01.766 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:01 smithi029 podman[164036]: 2024-03-29 04:59:01.445303816 +0000 UTC m=+0.370853902 container init ecfec7053880cfe4db32b22b78c5a02daaff500d55af5b9fb40d67e621350883 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:01.767 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:01 smithi029 podman[164036]: 2024-03-29 04:59:01.450981422 +0000 UTC m=+0.376531520 container start ecfec7053880cfe4db32b22b78c5a02daaff500d55af5b9fb40d67e621350883 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362) 2024-03-29T04:59:01.767 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:01 smithi029 podman[164036]: 2024-03-29 04:59:01.604278262 +0000 UTC m=+0.529828368 container attach ecfec7053880cfe4db32b22b78c5a02daaff500d55af5b9fb40d67e621350883 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux ) 2024-03-29T04:59:01.794 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:01.429+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:02.124 INFO:journalctl@ceph.osd.3.smithi029.stdout:Mar 29 04:59:01 smithi029 podman[164036]: 2024-03-29 04:59:01.76037247 +0000 UTC m=+0.685922576 container died ecfec7053880cfe4db32b22b78c5a02daaff500d55af5b9fb40d67e621350883 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-29T04:59:02.129 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:01.876+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:02.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:01.787+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:02.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:01 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:01.938+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:02.479 DEBUG:teuthology.orchestra.run.smithi029:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.3.service' 2024-03-29T04:59:02.527 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:59:02.527 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-29T04:59:02.528 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-29T04:59:02.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.4 2024-03-29T04:59:02.778 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:02.435+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:02.779 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:02 smithi099 systemd[1]: Stopping Ceph osd.4 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:02.779 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:02.770+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:03.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:02.880+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:03.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:02.951+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:03.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:02.855+0000 7fd2c7d64700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:59:03.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:02.855+0000 7fd2c7d64700 -1 osd.4 131 *** Got signal Terminated *** 2024-03-29T04:59:03.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:02 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:02.855+0000 7fd2c7d64700 -1 osd.4 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:59:03.804 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:03.453+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:04.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:03.875+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:04.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:03.943+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:04.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:03 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:03.796+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:04.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:04.317Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:04.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:04.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:04.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:04 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:04.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:04.811 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:04.409+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:05.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:04.866+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:05.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:04.945+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:05.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:04 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:04.803+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:05.830 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:05.422+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:05.830 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:05.422+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:06.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:05.822+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:06.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:05.899+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:06.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:05.899+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:06.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:05.851+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:06.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:05 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:05.851+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.101940+0000 front 2024-03-29T04:58:41.102021+0000 (oldest deadline 2024-03-29T04:59:05.801841+0000) 2024-03-29T04:59:06.784 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:06.399+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:06.785 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:06.399+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:07.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:06.776+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:07.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:06.871+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:32.301101+0000 front 2024-03-29T04:58:32.301089+0000 (oldest deadline 2024-03-29T04:58:56.400854+0000) 2024-03-29T04:59:07.130 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4[39440]: 2024-03-29T04:59:06.871+0000 7fd2c3360700 -1 osd.4 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.101940+0000 front 2024-03-29T04:58:41.102021+0000 (oldest deadline 2024-03-29T04:59:05.801841+0000) 2024-03-29T04:59:07.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:06.870+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:07.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:06 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:06.870+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:07.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:07.360+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:07.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:07.360+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:08.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:07.760+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:08.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:07.760+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:08.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:07.851+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:08.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:07 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:07.851+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:08.551 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:08 smithi099 podman[107011]: 2024-03-29 04:59:08.264406133 +0000 UTC m=+5.543721953 container died 1fab66997673190c50945ef145ba5edc341ee8b8efe62ad0029a7d1db42eb292 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2) 2024-03-29T04:59:08.551 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:08.316+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:08.551 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:08.316+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:08.802 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:08 smithi099 podman[107011]: 2024-03-29 04:59:08.644626785 +0000 UTC m=+5.923942596 container cleanup 1fab66997673190c50945ef145ba5edc341ee8b8efe62ad0029a7d1db42eb292 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-29T04:59:08.803 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:08 smithi099 bash[107011]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4 2024-03-29T04:59:09.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:08.794+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:09.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:08.794+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:09.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:08.840+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:09.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:08 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:08.840+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:09.131 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:08 smithi099 podman[107033]: 2024-03-29 04:59:08.896199942 +0000 UTC m=+0.626622725 container remove 1fab66997673190c50945ef145ba5edc341ee8b8efe62ad0029a7d1db42eb292 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.vendor=CentOS) 2024-03-29T04:59:09.511 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:09.291+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:09.512 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:09.291+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:09.512 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:09 smithi099 podman[107131]: 2024-03-29 04:59:09.155095253 +0000 UTC m=+0.017785854 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:59:09.512 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:09 smithi099 podman[107131]: 2024-03-29 04:59:09.27223052 +0000 UTC m=+0.134921094 container create ee61c769d3ed1d96c3aaad43835b6980fd5d812c7dd84548141dfbd48067de28 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-03-29T04:59:09.512 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:09 smithi099 podman[107131]: 2024-03-29 04:59:09.499918667 +0000 UTC m=+0.362609240 container init ee61c769d3ed1d96c3aaad43835b6980fd5d812c7dd84548141dfbd48067de28 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17) 2024-03-29T04:59:09.512 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:09 smithi099 podman[107131]: 2024-03-29 04:59:09.505049524 +0000 UTC m=+0.367740105 container start ee61c769d3ed1d96c3aaad43835b6980fd5d812c7dd84548141dfbd48067de28 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, ceph=True, RELEASE=quincy-4213e17, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:59:09.792 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:09 smithi099 podman[107131]: 2024-03-29 04:59:09.616767371 +0000 UTC m=+0.479457959 container attach ee61c769d3ed1d96c3aaad43835b6980fd5d812c7dd84548141dfbd48067de28 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-03-29T04:59:10.127 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:09.800+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:10.127 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:09.800+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:10.128 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:09.794+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:10.128 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:09 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:09.794+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:10.128 INFO:journalctl@ceph.osd.4.smithi099.stdout:Mar 29 04:59:09 smithi099 podman[107131]: 2024-03-29 04:59:09.785108085 +0000 UTC m=+0.647798663 container died ee61c769d3ed1d96c3aaad43835b6980fd5d812c7dd84548141dfbd48067de28 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-4-deactivate, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326) 2024-03-29T04:59:10.380 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:10.329+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:10.380 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:10.329+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:10.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.4.service' 2024-03-29T04:59:10.444 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:59:10.445 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-29T04:59:10.445 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-29T04:59:10.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.5 2024-03-29T04:59:10.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:10.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:10.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:10.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:10.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:10 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:10.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:10.784 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:10 smithi099 systemd[1]: Stopping Ceph osd.5 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:10.784 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:10.776+0000 7fcf12112700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:59:10.784 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:10.776+0000 7fcf12112700 -1 osd.5 131 *** Got signal Terminated *** 2024-03-29T04:59:10.784 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:10.776+0000 7fcf12112700 -1 osd.5 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:59:11.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:10.815+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:11.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:10.815+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:11.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:10.839+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:11.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:10 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:10.839+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:11.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:11 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:11.343+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:11.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:11 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:11.343+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:12.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:11 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:11.811+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:12.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:11 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:11.811+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:12.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:11 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:11.862+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:12.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:11 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:11.862+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:12.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:12.301+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:12.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:12.301+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:13.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:12.786+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:13.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:12.786+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:13.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:12.895+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:13.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:12.895+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:13.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:12 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:12.895+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.041569+0000 front 2024-03-29T04:58:51.041666+0000 (oldest deadline 2024-03-29T04:59:12.141424+0000) 2024-03-29T04:59:13.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:13.263+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:13.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:13.263+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:14.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:13.846+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:14.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:13.846+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:14.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:13.846+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.041569+0000 front 2024-03-29T04:58:51.041666+0000 (oldest deadline 2024-03-29T04:59:12.141424+0000) 2024-03-29T04:59:14.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:13.819+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:14.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:13 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:13.819+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:14.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:14.234+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:14.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:14.234+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:14.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:14.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:14.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:14.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:14.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:14 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:14.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:15.135 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:14.799+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:34.138998+0000 front 2024-03-29T04:58:34.139178+0000 (oldest deadline 2024-03-29T04:58:57.639049+0000) 2024-03-29T04:59:15.136 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:14.799+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:41.640584+0000 front 2024-03-29T04:58:41.640596+0000 (oldest deadline 2024-03-29T04:59:06.940487+0000) 2024-03-29T04:59:15.136 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5[45380]: 2024-03-29T04:59:14.799+0000 7fcf0f712700 -1 osd.5 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.041569+0000 front 2024-03-29T04:58:51.041666+0000 (oldest deadline 2024-03-29T04:59:12.141424+0000) 2024-03-29T04:59:15.136 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:14.821+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:15.136 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:14 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:14.821+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:15.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:15 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:15.237+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:15.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:15 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:15.237+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:16.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:15 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:15.807+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:16.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:15 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:15.807+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:16.630 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:16 smithi099 podman[107262]: 2024-03-29 04:59:16.350152264 +0000 UTC m=+5.691979554 container died 78420bf6aaa93cf9289574fbe2bae92e46386e1339588239554d8f0520e5746d (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17) 2024-03-29T04:59:16.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:16.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:16.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:16.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:16.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:16.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:17.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:16.842+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:17.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:16.842+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:17.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:16 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:16.842+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:17.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:16 smithi099 podman[107262]: 2024-03-29 04:59:16.723010363 +0000 UTC m=+6.064837684 container cleanup 78420bf6aaa93cf9289574fbe2bae92e46386e1339588239554d8f0520e5746d (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-29T04:59:17.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:16 smithi099 bash[107262]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5 2024-03-29T04:59:17.131 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:16 smithi099 podman[107286]: 2024-03-29 04:59:16.890718901 +0000 UTC m=+0.535679215 container remove 78420bf6aaa93cf9289574fbe2bae92e46386e1339588239554d8f0520e5746d (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:59:17.453 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:17 smithi099 podman[107383]: 2024-03-29 04:59:17.22847442 +0000 UTC m=+0.017804180 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:59:17.453 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:17 smithi099 podman[107383]: 2024-03-29 04:59:17.333487748 +0000 UTC m=+0.122817498 container create 145e0fdf31d937e59373549bb68eb3aff1cf7d8cbc7cfdad25c7d1253643a1fc (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-03-29T04:59:17.453 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:17.190+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:17.454 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:17.190+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:17.454 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:17.190+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:17.841 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:17 smithi099 podman[107383]: 2024-03-29 04:59:17.544526363 +0000 UTC m=+0.333856127 container init 145e0fdf31d937e59373549bb68eb3aff1cf7d8cbc7cfdad25c7d1253643a1fc (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-03-29T04:59:17.841 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:17 smithi099 podman[107383]: 2024-03-29 04:59:17.550203925 +0000 UTC m=+0.339533675 container start 145e0fdf31d937e59373549bb68eb3aff1cf7d8cbc7cfdad25c7d1253643a1fc (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-29T04:59:17.842 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:17 smithi099 podman[107383]: 2024-03-29 04:59:17.635978579 +0000 UTC m=+0.425308335 container attach 145e0fdf31d937e59373549bb68eb3aff1cf7d8cbc7cfdad25c7d1253643a1fc (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5-deactivate, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, RELEASE=quincy-4213e17) 2024-03-29T04:59:18.130 INFO:journalctl@ceph.osd.5.smithi099.stdout:Mar 29 04:59:17 smithi099 podman[107383]: 2024-03-29 04:59:17.834755931 +0000 UTC m=+0.624085675 container died 145e0fdf31d937e59373549bb68eb3aff1cf7d8cbc7cfdad25c7d1253643a1fc (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, io.buildah.version=1.33.5) 2024-03-29T04:59:18.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:17.891+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:18.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:17.891+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:18.131 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:17 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:17.891+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:18.480 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:18.188+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:18.480 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:18.188+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:18.480 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:18.188+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:18.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.5.service' 2024-03-29T04:59:18.548 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:59:18.548 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-29T04:59:18.548 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-29T04:59:18.548 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.6 2024-03-29T04:59:18.880 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 systemd[1]: Stopping Ceph osd.6 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:19.193 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:18.877+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:19.193 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:18.877+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:19.193 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:18.877+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:19.193 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:18.908+0000 7f6875e49700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:59:19.193 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:18.908+0000 7f6875e49700 -1 osd.6 131 *** Got signal Terminated *** 2024-03-29T04:59:19.194 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:18 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:18.908+0000 7f6875e49700 -1 osd.6 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:59:19.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:19.185+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:19.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:19.185+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:19.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:19.185+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:20.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:19.835+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:20.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:19.835+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:20.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:19 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:19.835+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:20.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:20.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:20.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:20.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:20.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:20 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:20.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:20.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:20.202+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:20.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:20.202+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:20.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:20.202+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:21.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:20.852+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:21.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:20.852+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:21.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:20.852+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:21.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:20 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:20.852+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.395700+0000 front 2024-03-29T04:58:56.395730+0000 (oldest deadline 2024-03-29T04:59:20.495655+0000) 2024-03-29T04:59:21.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:21.218+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:21.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:21.218+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:21.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:21.218+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:22.234 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:21.885+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:22.234 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:21.885+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:22.234 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:21.885+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:22.235 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:21 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:21.885+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.395700+0000 front 2024-03-29T04:58:56.395730+0000 (oldest deadline 2024-03-29T04:59:20.495655+0000) 2024-03-29T04:59:22.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:22.226+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:22.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:22.226+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:22.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:22.226+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:23.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:22.860+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:23.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:22.860+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:23.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:22.860+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:23.130 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:22 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:22.860+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.395700+0000 front 2024-03-29T04:58:56.395730+0000 (oldest deadline 2024-03-29T04:59:20.495655+0000) 2024-03-29T04:59:23.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:23.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:23.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:23.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:23.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:23.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:23.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:23.223+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:24.184 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:23.874+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:33.092644+0000 front 2024-03-29T04:58:33.092667+0000 (oldest deadline 2024-03-29T04:58:58.392374+0000) 2024-03-29T04:59:24.185 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:23.874+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:39.493563+0000 front 2024-03-29T04:58:39.493450+0000 (oldest deadline 2024-03-29T04:59:05.393429+0000) 2024-03-29T04:59:24.185 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:23.874+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:51.095129+0000 front 2024-03-29T04:58:51.095111+0000 (oldest deadline 2024-03-29T04:59:16.395113+0000) 2024-03-29T04:59:24.185 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:23 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6[51744]: 2024-03-29T04:59:23.874+0000 7f686f441700 -1 osd.6 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.395700+0000 front 2024-03-29T04:58:56.395730+0000 (oldest deadline 2024-03-29T04:59:20.495655+0000) 2024-03-29T04:59:24.616 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:24 smithi099 podman[107513]: 2024-03-29 04:59:24.333406927 +0000 UTC m=+5.558389103 container died 1f39d06fae80499b5491e883e172d481ff0110810366dc850330c485f32a0734 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-03-29T04:59:24.617 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:24.176+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:24.617 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:24.176+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:24.617 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:24.176+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:24.617 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:24 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:24.176+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:24.701 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:24.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:24.702 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:24.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:24.702 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:24 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:24.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:24.880 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:24 smithi099 podman[107513]: 2024-03-29 04:59:24.734021589 +0000 UTC m=+5.959003756 container cleanup 1f39d06fae80499b5491e883e172d481ff0110810366dc850330c485f32a0734 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:24.880 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:24 smithi099 bash[107513]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6 2024-03-29T04:59:25.291 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:24 smithi099 podman[107535]: 2024-03-29 04:59:24.993834063 +0000 UTC m=+0.659156368 container remove 1f39d06fae80499b5491e883e172d481ff0110810366dc850330c485f32a0734 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-29T04:59:25.291 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:25.192+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:25.291 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:25.192+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:25.291 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:25.192+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:25.292 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:25 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:25.192+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:25.621 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:25 smithi099 podman[107634]: 2024-03-29 04:59:25.187248986 +0000 UTC m=+0.018069956 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:59:25.621 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:25 smithi099 podman[107634]: 2024-03-29 04:59:25.303041319 +0000 UTC m=+0.133862269 container create 51f3ae5a017be9039bf954becab2de1bffd038cf972b47ca706337568073e225 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-29T04:59:25.621 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:25 smithi099 podman[107634]: 2024-03-29 04:59:25.505694172 +0000 UTC m=+0.336515127 container init 51f3ae5a017be9039bf954becab2de1bffd038cf972b47ca706337568073e225 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:25.621 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:25 smithi099 podman[107634]: 2024-03-29 04:59:25.51095531 +0000 UTC m=+0.341776269 container start 51f3ae5a017be9039bf954becab2de1bffd038cf972b47ca706337568073e225 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-03-29T04:59:25.622 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:25 smithi099 podman[107634]: 2024-03-29 04:59:25.613825637 +0000 UTC m=+0.444646603 container attach 51f3ae5a017be9039bf954becab2de1bffd038cf972b47ca706337568073e225 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-29T04:59:25.880 INFO:journalctl@ceph.osd.6.smithi099.stdout:Mar 29 04:59:25 smithi099 podman[107681]: 2024-03-29 04:59:25.81884053 +0000 UTC m=+0.017831358 container died 51f3ae5a017be9039bf954becab2de1bffd038cf972b47ca706337568073e225 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-29T04:59:26.341 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.148+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:26.341 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.148+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:26.342 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.148+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:26.342 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.148+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:26.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.6.service' 2024-03-29T04:59:26.400 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:59:26.400 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-29T04:59:26.401 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-29T04:59:26.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.7 2024-03-29T04:59:26.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 systemd[1]: Stopping Ceph osd.7 for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:27.130 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.723+0000 7fa80970c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:59:27.130 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.723+0000 7fa80970c700 -1 osd.7 131 *** Got signal Terminated *** 2024-03-29T04:59:27.130 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:26 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:26.723+0000 7fa80970c700 -1 osd.7 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:59:27.481 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:27 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:27.163+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:27.481 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:27 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:27.163+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:27.481 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:27 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:27.163+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:27.481 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:27 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:27.163+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:28.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:28.172+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:28.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:28.172+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:28.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:28.172+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:28.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:28 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:28.172+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:29.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:29.203+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:29.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:29.203+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:29.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:29.203+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:29.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:29.203+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:29.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:29 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:29.203+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.99:6804 osd.4 since back 2024-03-29T04:59:06.387421+0000 front 2024-03-29T04:59:06.387454+0000 (oldest deadline 2024-03-29T04:59:28.687347+0000) 2024-03-29T04:59:30.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:30.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:30.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:30.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:30.450 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:30 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:30.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:30.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:30.161+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:30.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:30.161+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:30.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:30.161+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:30.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:30.161+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:30.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:30 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:30.161+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.99:6804 osd.4 since back 2024-03-29T04:59:06.387421+0000 front 2024-03-29T04:59:06.387454+0000 (oldest deadline 2024-03-29T04:59:28.687347+0000) 2024-03-29T04:59:31.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:31.171+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6806 osd.0 since back 2024-03-29T04:58:31.482114+0000 front 2024-03-29T04:58:31.482101+0000 (oldest deadline 2024-03-29T04:58:56.782038+0000) 2024-03-29T04:59:31.630 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:31.171+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6814 osd.1 since back 2024-03-29T04:58:38.983615+0000 front 2024-03-29T04:58:38.983638+0000 (oldest deadline 2024-03-29T04:59:04.883526+0000) 2024-03-29T04:59:31.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:31.171+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6822 osd.2 since back 2024-03-29T04:58:50.585507+0000 front 2024-03-29T04:58:50.585520+0000 (oldest deadline 2024-03-29T04:59:15.285417+0000) 2024-03-29T04:59:31.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:31.171+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.29:6830 osd.3 since back 2024-03-29T04:58:56.986553+0000 front 2024-03-29T04:58:56.986582+0000 (oldest deadline 2024-03-29T04:59:22.286492+0000) 2024-03-29T04:59:31.631 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:31 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7[58008]: 2024-03-29T04:59:31.171+0000 7fa804507700 -1 osd.7 131 heartbeat_check: no reply from 172.21.15.99:6804 osd.4 since back 2024-03-29T04:59:06.387421+0000 front 2024-03-29T04:59:06.387454+0000 (oldest deadline 2024-03-29T04:59:28.687347+0000) 2024-03-29T04:59:32.504 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:32 smithi099 podman[107765]: 2024-03-29 04:59:32.206163857 +0000 UTC m=+5.594843632 container died 6adf0209fbb473b777f1df1e8e3e1c844bfcda8ee678c69674294012d1c49a35 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7, org.label-schema.build-date=20240326, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-03-29T04:59:32.845 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:32 smithi099 podman[107765]: 2024-03-29 04:59:32.595629154 +0000 UTC m=+5.984308961 container cleanup 6adf0209fbb473b777f1df1e8e3e1c844bfcda8ee678c69674294012d1c49a35 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:32.845 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:32 smithi099 bash[107765]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7 2024-03-29T04:59:33.130 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:32 smithi099 podman[107798]: 2024-03-29 04:59:32.838399651 +0000 UTC m=+0.626769575 container remove 6adf0209fbb473b777f1df1e8e3e1c844bfcda8ee678c69674294012d1c49a35 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-29T04:59:33.449 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:33 smithi099 podman[107895]: 2024-03-29 04:59:33.063907009 +0000 UTC m=+0.017628341 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:59:33.449 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:33 smithi099 podman[107895]: 2024-03-29 04:59:33.164317482 +0000 UTC m=+0.118038794 container create e47f422fb504064dab9c87de46d093643c80d20aa0962fe884c7c8a9331fb9c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-29T04:59:33.449 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:33 smithi099 podman[107895]: 2024-03-29 04:59:33.350337498 +0000 UTC m=+0.304058821 container init e47f422fb504064dab9c87de46d093643c80d20aa0962fe884c7c8a9331fb9c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-29T04:59:33.449 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:33 smithi099 podman[107895]: 2024-03-29 04:59:33.355570349 +0000 UTC m=+0.309291671 container start e47f422fb504064dab9c87de46d093643c80d20aa0962fe884c7c8a9331fb9c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-29T04:59:33.449 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:33 smithi099 podman[107895]: 2024-03-29 04:59:33.441882143 +0000 UTC m=+0.395603464 container attach e47f422fb504064dab9c87de46d093643c80d20aa0962fe884c7c8a9331fb9c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7-deactivate, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17) 2024-03-29T04:59:33.880 INFO:journalctl@ceph.osd.7.smithi099.stdout:Mar 29 04:59:33 smithi099 podman[107895]: 2024-03-29 04:59:33.641654042 +0000 UTC m=+0.595375366 container died e47f422fb504064dab9c87de46d093643c80d20aa0962fe884c7c8a9331fb9c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-osd-7-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=quincy-4213e17) 2024-03-29T04:59:34.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@osd.7.service' 2024-03-29T04:59:34.302 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:59:34.303 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-29T04:59:34.303 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-03-29T04:59:34.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a 2024-03-29T04:59:34.630 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 systemd[1]: Stopping Ceph prometheus.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:34.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:34.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:34.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:34.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi029.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.29:8443: connect: connection refused" 2024-03-29T04:59:34.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:34 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:34.320Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2024-03-29T04:59:34.956 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:59:34.956 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:59:34.956 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:59:34.956 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.659Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.663Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.663Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a[82537]: ts=2024-03-29T04:59:34.663Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:59:34.957 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:34 smithi099 podman[108026]: 2024-03-29 04:59:34.669710355 +0000 UTC m=+0.149429260 container died cfdb8be5d010c429483d833895153b3a88acb559dccd7c0bb397180ee31c59b7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:35.314 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:35 smithi099 podman[108026]: 2024-03-29 04:59:35.05669654 +0000 UTC m=+0.536415436 container cleanup cfdb8be5d010c429483d833895153b3a88acb559dccd7c0bb397180ee31c59b7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:35.315 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Mar 29 04:59:35 smithi099 bash[108026]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:59:35.476 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-37319ec8-ed84-11ee-95d0-87774f69a715@prometheus.a.service' 2024-03-29T04:59:35.513 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:59:35.513 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-03-29T04:59:35.513 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-29T04:59:37.200 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:37 smithi029 systemd[1]: Stopping Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:37.660 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:37 smithi029 ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a[91557]: ts=2024-03-29T04:59:37.287Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:59:37.661 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:37 smithi029 podman[164472]: 2024-03-29 04:59:37.328385331 +0000 UTC m=+0.140009132 container died 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:37.950 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:37 smithi029 podman[164472]: 2024-03-29 04:59:37.847083059 +0000 UTC m=+0.658706848 container cleanup 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:37.950 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:37 smithi029 bash[164472]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:59:38.411 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:38 smithi029 podman[164492]: 2024-03-29 04:59:38.057897242 +0000 UTC m=+0.727082452 container remove 6fb7bc14815462e4e928f96e5f3e4fcbfb3354b645c7db3687c2add630232193 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:38.412 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:38 smithi029 podman[164492]: 2024-03-29 04:59:38.144011184 +0000 UTC m=+0.813196406 volume remove 71fa853b2d8b3145772fd3817d2232a969c4a29fa370f7dcdf252f3bc2b8816a 2024-03-29T04:59:38.700 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:38 smithi029 systemd[1]: Stopping Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:38.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:38 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:59:38.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:38 smithi029 systemd[1]: Stopped Ceph alertmanager.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:59:38.700 INFO:journalctl@ceph.alertmanager.a.smithi029.stdout:Mar 29 04:59:38 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@alertmanager.a.service: Consumed 2.819s CPU time. 2024-03-29T04:59:39.197 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:38 smithi029 podman[164603]: 2024-03-29 04:59:38.87677295 +0000 UTC m=+0.135047543 container died 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:39.450 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 podman[164603]: 2024-03-29 04:59:39.382258887 +0000 UTC m=+0.640533473 container cleanup 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:39.450 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 bash[164603]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a 2024-03-29T04:59:39.923 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 podman[164622]: 2024-03-29 04:59:39.63527518 +0000 UTC m=+0.761534930 container remove 9026c9798e07ceee8c1ee939f3dcb26e5b50a2fe7837cd49aef45385e69fd8f8 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:59:39.923 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:59:39.923 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-29T04:59:39.923 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 systemd[1]: Stopped Ceph node-exporter.a for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:59:39.923 INFO:journalctl@ceph.node-exporter.a.smithi029.stdout:Mar 29 04:59:39 smithi029 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.a.service: Consumed 4.163s CPU time. 2024-03-29T04:59:50.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-29T04:59:52.381 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:51 smithi099 systemd[1]: Stopping Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715... 2024-03-29T04:59:52.381 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 podman[108483]: 2024-03-29 04:59:52.222802171 +0000 UTC m=+0.107432181 container died 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:59:52.880 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 podman[108483]: 2024-03-29 04:59:52.550556537 +0000 UTC m=+0.435186539 container cleanup 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:59:52.880 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 bash[108483]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b 2024-03-29T04:59:52.880 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 podman[108501]: 2024-03-29 04:59:52.710110907 +0000 UTC m=+0.488236263 container remove 52f151f284613d51fe100e4b38f2ffdf63dd65cc42cbe6f585741f88ba4ec0d6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-37319ec8-ed84-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:59:52.880 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:59:53.340 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-29T04:59:53.340 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 systemd[1]: Stopped Ceph node-exporter.b for 37319ec8-ed84-11ee-95d0-87774f69a715. 2024-03-29T04:59:53.340 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Mar 29 04:59:52 smithi099 systemd[1]: ceph-37319ec8-ed84-11ee-95d0-87774f69a715@node-exporter.b.service: Consumed 3.720s CPU time. 2024-03-29T04:59:58.959 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:59:58.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:59:59.024 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-29T04:59:59.026 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/remote/smithi029/crash 2024-03-29T04:59:59.027 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/crash -- . 2024-03-29T04:59:59.082 INFO:teuthology.orchestra.run.smithi029.stderr:tar: /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-29T04:59:59.082 INFO:teuthology.orchestra.run.smithi029.stderr:tar: Error is not recoverable: exiting now 2024-03-29T04:59:59.084 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/remote/smithi099/crash 2024-03-29T04:59:59.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/crash -- . 2024-03-29T04:59:59.123 INFO:teuthology.orchestra.run.smithi099.stderr:tar: /var/lib/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-29T04:59:59.123 INFO:teuthology.orchestra.run.smithi099.stderr:tar: Error is not recoverable: exiting now 2024-03-29T04:59:59.124 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-29T04:59:59.125 DEBUG:teuthology.orchestra.run.smithi029:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-29T04:59:59.277 INFO:teuthology.orchestra.run.smithi029.stdout:1711686588.8149009 mon.a (mon.0) 512 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:59:59.277 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-29T04:59:59.278 INFO:tasks.cephadm:Compressing logs... 2024-03-29T04:59:59.278 DEBUG:teuthology.orchestra.run.smithi029:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-29T04:59:59.280 DEBUG:teuthology.orchestra.run.smithi099:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-29T04:59:59.312 INFO:teuthology.orchestra.run.smithi099.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-29T04:59:59.313 INFO:teuthology.orchestra.run.smithi099.stderr:: No such file or directory 2024-03-29T04:59:59.313 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.6.log 2024-03-29T04:59:59.313 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-29T04:59:59.313 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.4.log 2024-03-29T04:59:59.313 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.6.log: /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.7.log 2024-03-29T04:59:59.314 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mds.foo.smithi099.jhvkhg.log 2024-03-29T04:59:59.314 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.5.log 2024-03-29T04:59:59.314 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mds.foo.smithi099.jhvkhg.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mgr.x.log 2024-03-29T04:59:59.314 INFO:teuthology.orchestra.run.smithi099.stderr: 62.3% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mds.foo.smithi099.jhvkhg.log.gz 2024-03-29T04:59:59.314 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-29T04:59:59.314 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log 2024-03-29T04:59:59.315 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.cephadm.log: 84.1% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-29T04:59:59.315 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-29T04:59:59.319 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log: 89.5% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-29T04:59:59.319 INFO:teuthology.orchestra.run.smithi099.stderr:gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.b.log 2024-03-29T04:59:59.320 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-volume.log: 87.5% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-29T04:59:59.322 INFO:teuthology.orchestra.run.smithi029.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-03-29T04:59:59.322 INFO:teuthology.orchestra.run.smithi029.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-29T04:59:59.322 INFO:teuthology.orchestra.run.smithi029.stderr:gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.a.log 2024-03-29T04:59:59.322 INFO:teuthology.orchestra.run.smithi029.stderr:gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-29T04:59:59.322 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.1.log 2024-03-29T04:59:59.322 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.3.log 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mgr.y.log 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mds.foo.smithi029.gsovzu.log 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.0.log 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mds.foo.smithi029.gsovzu.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr: 82.0% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mds.foo.smithi029.gsovzu.log.gz 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log 2024-03-29T04:59:59.323 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.2.log 2024-03-29T04:59:59.324 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-29T04:59:59.324 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.c.log 2024-03-29T04:59:59.325 INFO:teuthology.orchestra.run.smithi029.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-volume.log: /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.c.log: 92.7% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-29T04:59:59.325 INFO:teuthology.orchestra.run.smithi099.stderr:/var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.b.log: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-29T04:59:59.329 INFO:teuthology.orchestra.run.smithi099.stderr: 91.2% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-29T04:59:59.380 INFO:teuthology.orchestra.run.smithi099.stderr: 89.5% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mgr.x.log.gz 2024-03-29T04:59:59.486 INFO:teuthology.orchestra.run.smithi029.stderr: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-29T04:59:59.707 INFO:teuthology.orchestra.run.smithi099.stderr: 92.0% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.b.log.gz 2024-03-29T04:59:59.800 INFO:teuthology.orchestra.run.smithi029.stderr: 90.5% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-29T04:59:59.815 INFO:teuthology.orchestra.run.smithi029.stderr: 93.6% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-29T05:00:00.208 INFO:teuthology.orchestra.run.smithi029.stderr: 91.0% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-29T05:00:02.483 INFO:teuthology.orchestra.run.smithi029.stderr: 88.6% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mgr.y.log.gz 2024-03-29T05:00:04.404 INFO:teuthology.orchestra.run.smithi029.stderr: 91.6% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.c.log.gz 2024-03-29T05:00:09.143 INFO:teuthology.orchestra.run.smithi029.stderr: 90.3% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-mon.a.log.gz 2024-03-29T05:00:33.807 INFO:teuthology.orchestra.run.smithi099.stderr: 93.8% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.6.log.gz 2024-03-29T05:00:38.440 INFO:teuthology.orchestra.run.smithi099.stderr: 93.6% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.4.log.gz 2024-03-29T05:00:38.912 INFO:teuthology.orchestra.run.smithi099.stderr: 93.6% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.7.log.gz 2024-03-29T05:00:54.948 INFO:teuthology.orchestra.run.smithi099.stderr: 93.2% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.5.log.gz 2024-03-29T05:00:54.950 INFO:teuthology.orchestra.run.smithi099.stderr: 2024-03-29T05:00:54.950 INFO:teuthology.orchestra.run.smithi099.stderr:real 0m55.654s 2024-03-29T05:00:54.950 INFO:teuthology.orchestra.run.smithi099.stderr:user 2m42.884s 2024-03-29T05:00:54.950 INFO:teuthology.orchestra.run.smithi099.stderr:sys 0m6.192s 2024-03-29T05:00:57.806 INFO:teuthology.orchestra.run.smithi029.stderr: 93.2% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.2.log.gz 2024-03-29T05:00:59.500 INFO:teuthology.orchestra.run.smithi029.stderr: 93.3% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.0.log.gz 2024-03-29T05:02:10.095 INFO:teuthology.orchestra.run.smithi029.stderr: 93.5% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.1.log.gz 2024-03-29T05:02:51.675 INFO:teuthology.orchestra.run.smithi029.stderr: 92.9% -- replaced with /var/log/ceph/37319ec8-ed84-11ee-95d0-87774f69a715/ceph-osd.3.log.gz 2024-03-29T05:02:51.677 INFO:teuthology.orchestra.run.smithi029.stderr: 2024-03-29T05:02:51.677 INFO:teuthology.orchestra.run.smithi029.stderr:real 2m52.382s 2024-03-29T05:02:51.677 INFO:teuthology.orchestra.run.smithi029.stderr:user 3m55.125s 2024-03-29T05:02:51.677 INFO:teuthology.orchestra.run.smithi029.stderr:sys 0m8.760s 2024-03-29T05:02:51.678 INFO:tasks.cephadm:Archiving logs... 2024-03-29T05:02:51.678 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/remote/smithi029/log 2024-03-29T05:02:51.679 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-29T05:03:26.672 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/remote/smithi099/log 2024-03-29T05:03:26.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-29T05:03:51.738 INFO:tasks.cephadm:Removing cluster... 2024-03-29T05:03:51.738 DEBUG:teuthology.orchestra.run.smithi029:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 --force 2024-03-29T05:03:52.537 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 37319ec8-ed84-11ee-95d0-87774f69a715 --force 2024-03-29T05:03:53.267 INFO:tasks.cephadm:Removing cephadm ... 2024-03-29T05:03:53.268 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-29T05:03:53.285 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-29T05:03:53.300 INFO:tasks.cephadm:Teardown complete 2024-03-29T05:03:53.301 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-29T05:03:53.312 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-29T05:03:53.312 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-29T05:03:53.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-29T05:03:53.410 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-29T05:03:53.419 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-29T05:03:53.420 DEBUG:teuthology.orchestra.run.smithi029:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T05:03:53.422 DEBUG:teuthology.orchestra.run.smithi099:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T05:03:53.435 INFO:teuthology.orchestra.run.smithi029.stderr:bash: line 1: ntpq: command not found 2024-03-29T05:03:53.436 INFO:teuthology.orchestra.run.smithi099.stderr:bash: line 1: ntpq: command not found 2024-03-29T05:03:53.440 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T05:03:53.441 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2024-03-29T05:03:53.441 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 55 -509us[ -509us] +/- 89ms 2024-03-29T05:03:53.441 INFO:teuthology.orchestra.run.smithi099.stdout:^* hv02.front.sepia.ceph.com 3 6 377 56 -1871us[-1863us] +/- 69ms 2024-03-29T05:03:53.441 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 +2081us[+2081us] +/- 65ms 2024-03-29T05:03:53.441 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T05:03:53.455 INFO:teuthology.orchestra.run.smithi029.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T05:03:53.455 INFO:teuthology.orchestra.run.smithi029.stdout:=============================================================================== 2024-03-29T05:03:53.455 INFO:teuthology.orchestra.run.smithi029.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 51 -473us[ -473us] +/- 90ms 2024-03-29T05:03:53.455 INFO:teuthology.orchestra.run.smithi029.stdout:^* hv02.front.sepia.ceph.com 3 6 377 53 -1855us[-1854us] +/- 69ms 2024-03-29T05:03:53.456 INFO:teuthology.orchestra.run.smithi029.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 +2100us[+2101us] +/- 65ms 2024-03-29T05:03:53.456 INFO:teuthology.orchestra.run.smithi029.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T05:03:53.457 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-29T05:03:53.469 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-29T05:03:53.470 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-29T05:03:53.480 DEBUG:teuthology.orchestra.run.smithi029:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-29T05:03:53.611 DEBUG:teuthology.orchestra.run.smithi099:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-29T05:03:53.691 DEBUG:teuthology.orchestra.run.smithi029:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T05:03:53.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:03:53.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T05:03:53.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:03:53.754 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-29T05:03:53.764 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-29T05:03:53.774 INFO:teuthology.task.internal:Duration was 2782.756960 seconds 2024-03-29T05:03:53.775 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-29T05:03:53.784 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-29T05:03:53.784 DEBUG:teuthology.orchestra.run.smithi029:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-29T05:03:53.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-29T05:03:53.840 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T05:03:53.844 INFO:teuthology.orchestra.run.smithi029.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T05:03:54.252 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-29T05:03:54.252 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi029.front.sepia.ceph.com 2024-03-29T05:03:54.253 DEBUG:teuthology.orchestra.run.smithi029:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-29T05:03:54.273 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi099.front.sepia.ceph.com 2024-03-29T05:03:54.273 DEBUG:teuthology.orchestra.run.smithi099:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-29T05:03:54.307 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-29T05:03:54.307 DEBUG:teuthology.orchestra.run.smithi029:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-29T05:03:54.315 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-29T05:03:54.375 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-29T05:03:54.376 DEBUG:teuthology.orchestra.run.smithi029:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-29T05:03:54.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-29T05:03:54.856 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-29T05:03:54.867 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-29T05:03:54.867 DEBUG:teuthology.orchestra.run.smithi029:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-29T05:03:54.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-29T05:03:54.930 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-29T05:03:54.941 DEBUG:teuthology.orchestra.run.smithi029:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:03:54.945 DEBUG:teuthology.orchestra.run.smithi099:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:03:54.982 INFO:teuthology.orchestra.run.smithi029.stdout:kernel.core_pattern = core 2024-03-29T05:03:54.996 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = core 2024-03-29T05:03:55.079 DEBUG:teuthology.orchestra.run.smithi029:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:03:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:03:55.134 DEBUG:teuthology.orchestra.run.smithi099:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:03:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:03:55.149 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-29T05:03:55.160 INFO:teuthology.task.internal:Transferring archived files... 2024-03-29T05:03:55.161 DEBUG:teuthology.misc:Transferring archived files from smithi029:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/remote/smithi029 2024-03-29T05:03:55.161 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-29T05:03:55.227 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108/remote/smithi099 2024-03-29T05:03:55.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-29T05:03:55.268 INFO:teuthology.task.internal:Removing archive directory... 2024-03-29T05:03:55.269 DEBUG:teuthology.orchestra.run.smithi029:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-29T05:03:55.271 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-29T05:03:55.316 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-29T05:03:55.328 INFO:teuthology.task.internal:Not uploading archives. 2024-03-29T05:03:55.328 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-29T05:03:55.339 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-29T05:03:55.339 DEBUG:teuthology.orchestra.run.smithi029:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-29T05:03:55.341 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-29T05:03:55.353 INFO:teuthology.orchestra.run.smithi029.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 29 05:03 /home/ubuntu/cephtest 2024-03-29T05:03:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 29 05:03 /home/ubuntu/cephtest 2024-03-29T05:03:55.371 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-29T05:03:55.382 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-29T05:03:55.403 INFO:teuthology.nuke:Checking targets against current locks 2024-03-29T05:03:55.428 DEBUG:teuthology.nuke:shortname: smithi029 2024-03-29T05:03:55.428 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T05:03:55.456 DEBUG:teuthology.nuke:shortname: smithi099 2024-03-29T05:03:55.457 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T05:03:55.460 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi029.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:08:34.354862', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJI7iNJ3YgHdkr+lBiW/B2AtbNJP5K6CIfLSCFpmyc0a6Ts/vecmiRiNTrYQ1Nnknsr4Ybi3eegCYRcXWF9qJO4='} 2024-03-29T05:03:55.479 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629108', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:08:34.356336', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMTGQ7qxV9Ue7P8GrTfDR8/kOj2hexkGczvRuhjHopL6xl53lYWmHrQvo2QWmC5hZK0bEwaEvVyO0/HgVaYXNIg='} 2024-03-29T05:03:55.507 INFO:teuthology.orchestra.console.smithi029:Power off 2024-03-29T05:03:55.507 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-29T05:03:55.533 DEBUG:teuthology.orchestra.console.smithi029:power off output: Chassis Power Control: Down/Off 2024-03-29T05:03:55.533 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:03:55.549 INFO:teuthology.orchestra.console.smithi099:Power off 2024-03-29T05:03:55.550 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-29T05:03:55.560 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is on 2024-03-29T05:03:55.576 DEBUG:teuthology.orchestra.console.smithi099:power off output: Chassis Power Control: Down/Off 2024-03-29T05:03:55.576 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:03:55.603 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2024-03-29T05:03:59.561 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:03:59.604 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:03:59.673 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is on 2024-03-29T05:03:59.717 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2024-03-29T05:04:03.678 DEBUG:teuthology.orchestra.console.smithi029:pexpect command: ipmitool -H smithi029.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:04:03.717 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:04:03.790 DEBUG:teuthology.orchestra.console.smithi029:check power output: Chassis Power is off 2024-03-29T05:04:03.829 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is off 2024-03-29T05:04:03.890 INFO:teuthology.orchestra.console.smithi029:Power off completed 2024-03-29T05:04:03.930 INFO:teuthology.orchestra.console.smithi099:Power off completed 2024-03-29T05:04:04.035 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 2782.7569601535797 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=4905ab564e4b4074bd94e9b52c1da1df status: fail success: false 2024-03-29T05:04:04.035 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-29T05:04:04.113 INFO:teuthology.run:FAIL